From 2822a483e30b5a258cb125c89cb6b0b34e0dabff Mon Sep 17 00:00:00 2001 From: Harrison Deng Date: Fri, 21 Feb 2025 05:37:56 +0000 Subject: [PATCH] Initial attempt at switching to a conda based build environment --- .devcontainer/Dockerfile | 11 +++ .devcontainer/devcontainer.json | 19 ++-- .devcontainer/noop.txt | 3 + .gitignore | 159 ++------------------------------ .vscode/extensions.json | 5 + Jenkinsfile | 10 +- autobigs-engine/meta.yaml | 44 +++++++++ environment.yml | 15 +++ requirements.txt | 8 -- scripts/patch_recipe.py | 103 +++++++++++++++++++++ 10 files changed, 202 insertions(+), 175 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/noop.txt create mode 100644 .vscode/extensions.json create mode 100644 autobigs-engine/meta.yaml create mode 100644 environment.yml delete mode 100644 requirements.txt create mode 100644 scripts/patch_recipe.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..2738918 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,11 @@ +FROM mcr.microsoft.com/devcontainers/anaconda:1-3 + +# Copy environment.yml (if found) to a temp location so we update the environment. Also +# copy "noop.txt" so the COPY instruction does not fail if no environment.yml exists. +COPY environment.yml* .devcontainer/noop.txt /tmp/conda-tmp/ +RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; fi \ + && rm -rf /tmp/conda-tmp + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c58d0e4..70e91b2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,9 +1,11 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the -// README at: https://github.com/devcontainers/templates/tree/main/src/python +// README at: https://github.com/devcontainers/templates/tree/main/src/anaconda { - "name": "Python 3", - // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye", + "name": "Anaconda (Python 3)", + "build": { + "context": "..", + "dockerfile": "Dockerfile" + } // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, @@ -12,14 +14,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip3 install --user -r requirements.txt", - "customizations": { - "vscode": { - "extensions": [ - "mechatroner.rainbow-csv" - ] - } - } + // "postCreateCommand": "python --version", // Configure tool-specific properties. // "customizations": {}, diff --git a/.devcontainer/noop.txt b/.devcontainer/noop.txt new file mode 100644 index 0000000..dde8dc3 --- /dev/null +++ b/.devcontainer/noop.txt @@ -0,0 +1,3 @@ +This file copied into the container along with environment.yml* from the parent +folder. This file is included to prevents the Dockerfile COPY instruction from +failing if no environment.yml is found. \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2a36fc4..2438221 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig -# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,svelte,python,linux,node -# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,svelte,python,linux,node +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,linux,python +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,linux,python ### Linux ### *~ @@ -17,146 +17,6 @@ # .nfs files are created when an open file is removed but is still being accessed .nfs* -### Node ### -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* -.pnpm-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Microbundle cache -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files -.env -.env.development.local -.env.test.local -.env.production.local -.env.local - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp and cache directory -.temp - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# yarn v2 -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.* - -### Node Patch ### -# Serverless Webpack directories -.webpack/ - -# Optional stylelint cache - -# SvelteKit build / generate output -.svelte-kit - ### Python ### # Byte-compiled / optimized / DLL files __pycache__/ @@ -202,6 +62,7 @@ htmlcov/ .nox/ .coverage .coverage.* +.cache nosetests.xml coverage.xml *.cover @@ -215,6 +76,7 @@ cover/ *.pot # Django stuff: +*.log local_settings.py db.sqlite3 db.sqlite3-journal @@ -278,6 +140,7 @@ celerybeat.pid *.sage.py # Environments +.env .venv env/ venv/ @@ -326,13 +189,6 @@ poetry.toml # LSP config files pyrightconfig.json -### Svelte ### -# gitignore template for the SvelteKit, frontend web component framework -# website: https://kit.svelte.dev/ - -.svelte-kit/ -package - ### VisualStudioCode ### .vscode/* !.vscode/settings.json @@ -352,9 +208,8 @@ package .history .ionide -# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,svelte,python,linux,node +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,linux,python # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) -output -*.private.* \ No newline at end of file +conda-bld \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..1bddac4 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "piotrpalarz.vscode-gitignore-generator" + ] +} \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index fb4051e..3fad1de 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,7 @@ pipeline { stages { stage("install") { steps { - sh 'python -m pip install -r requirements.txt' + sh 'conda env update -n base -f environment.yml' } } stage("unit tests") { @@ -22,11 +22,14 @@ pipeline { stage("build") { steps { sh "python -m build" + sh "grayskull pypi dist/*.tar.gz --maintainers 'Harrison Deng'" + sh "python scripts/patch_recipe.py" + sh 'conda build autobigs-engine -c bioconda --output-folder conda-bld --verify' } } stage("archive") { steps { - archiveArtifacts artifacts: 'dist/*.tar.gz, dist/*.whl', fingerprint: true, followSymlinks: false, onlyIfSuccessful: true + archiveArtifacts artifacts: 'dist/*.tar.gz, dist/*.whl, conda-bld/**/*.conda', fingerprint: true, followSymlinks: false, onlyIfSuccessful: true } } stage("publish") { @@ -36,7 +39,8 @@ pipeline { CREDS = credentials('username-password-rs-git') } steps { - sh returnStatus: true, script: 'python -m twine upload --repository-url https://git.reslate.systems/api/packages/ydeng/pypi -u ${CREDS_USR} -p ${CREDS_PSW} --non-interactive --disable-progress-bar --verbose dist/*' + sh 'python -m twine upload --repository-url https://git.reslate.systems/api/packages/ydeng/pypi -u ${CREDS_USR} -p ${CREDS_PSW} --non-interactive --disable-progress-bar --verbose dist/*' + sh 'curl --user ${CREDS_USR}:${CRED_PSW} --upload-file conda-bld/**/*.conda https://git.reslate.systems/api/packages/${CRED_USR}/conda/$(basename conda-bld/**/*.conda)' } } stage ("pypi.org") { diff --git a/autobigs-engine/meta.yaml b/autobigs-engine/meta.yaml new file mode 100644 index 0000000..c27ddbc --- /dev/null +++ b/autobigs-engine/meta.yaml @@ -0,0 +1,44 @@ +{% set name = "autoBIGS.engine" %} +{% set version = "0.12.1.dev1+gb8cebb8.d20250221" %} + +package: + name: {{ name|lower|replace(".", "-") }} + version: {{ version }} + +source: + url: file:///workspaces/autoBIGS.engine/dist/autobigs_engine-0.12.1.dev1%2Bgb8cebb8.d20250221.tar.gz + sha256: c86441b94f935cfa414ff28ca4c026a070e0fb15988ea3bb7d1a942859a09b16 + +build: + noarch: python + script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation + number: 0 + run_exports: + - {{ pin_subpackage( name|lower|replace(".", "-"), max_pin="x.x") }} +requirements: + host: + - python >=3.12 + - setuptools >=64 + - setuptools-scm >=8 + - pip + run: + - python >=3.12 + - biopython ==1.85 + - aiohttp ==3.11.* + +test: + imports: + - autobigs + commands: + - pip check + requires: + - pip + +about: + summary: A library to rapidly fetch fetch MLST profiles given sequences for various diseases. + license: GPL-3.0-or-later + license_file: LICENSE + home: https://github.com/Syph-and-VPD-Lab/autoBIGS.engine +extra: + recipe-maintainers: + - Harrison Deng diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..cb1fa6e --- /dev/null +++ b/environment.yml @@ -0,0 +1,15 @@ +name: ci +channels: + - bioconda + - conda-forge +dependencies: + - aiohttp==3.11.* + - biopython==1.85 + - pytest + - pytest-asyncio + - python-build + - conda-build + - twine + - setuptools_scm + - pytest-cov + - grayskull \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 0b8be9b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -aiohttp[speedups]==3.11.* -biopython==1.85 -pytest -pytest-asyncio -build -twine -setuptools_scm -pytest-cov \ No newline at end of file diff --git a/scripts/patch_recipe.py b/scripts/patch_recipe.py new file mode 100644 index 0000000..e31403b --- /dev/null +++ b/scripts/patch_recipe.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +import argparse +from os import fdopen, path +import os +import re +import shutil +from sys import argv +import tempfile + +INDENTATION = " " +GRAYSKULL_OUTPUT_PATH = "autoBIGS.engine" +RUN_EXPORTED_VALUE = r'{{ pin_subpackage( name|lower|replace(".", "-"), max_pin="x.x") }}' +LICENSE_SUFFIX = "-or-later" +HOME_PAGE = "https://github.com/Syph-and-VPD-Lab/autoBIGS.engine" + +def _calc_indentation(line: str): + return len(re.findall(INDENTATION, line.split(line.strip())[0])) if line != "\n" else 0 + +def read_grayskull_output(): + original_recipe = path.abspath(GRAYSKULL_OUTPUT_PATH) + original_meta = path.join(original_recipe, "meta.yaml") + meta_file = open(original_meta) + lines = meta_file.readlines() + meta_file.close() + return lines + +def update_naming_scheme(lines): + modified_lines = [] + for line in lines: + matches = re.finditer(r"\{\{\s*name\|lower()\s+\}\}", line) + modified_line = line + for match in matches: + modified_line = modified_line[:match.start(1)] + r'|replace(".", "-")' + modified_line[match.end(1):] + modified_lines.append(modified_line) + return modified_lines + +def inject_run_exports(lines: list[str]): + package_indent = False + modified_lines = [] + for line in lines: + indentation_count = _calc_indentation(line) + if line == "build:\n" and indentation_count == 0: + package_indent = True + modified_lines.append(line) + elif package_indent and indentation_count == 0: + modified_lines.append(INDENTATION*1 + "run_exports:\n") + modified_lines.append(INDENTATION*2 + "- " + RUN_EXPORTED_VALUE + "\n") + package_indent = False + else: + modified_lines.append(line) + return modified_lines + +def suffix_license(lines: list[str]): + about_indent = False + modified_lines = [] + for line in lines: + indentation_count = _calc_indentation(line) + if line == "about:\n" and indentation_count == 0: + about_indent = True + modified_lines.append(line) + elif about_indent and indentation_count == 1 and line.lstrip().startswith("license:"): + modified_lines.append(line.rstrip() + LICENSE_SUFFIX + "\n") + about_indent = False + else: + modified_lines.append(line) + return modified_lines + +def inject_home_page(lines: list[str]): + about_indent = False + modified_lines = [] + for line in lines: + indentation_count = _calc_indentation(line) + if line == "about:\n" and indentation_count == 0: + about_indent = True + modified_lines.append(line) + elif about_indent and indentation_count == 0: + modified_lines.append(INDENTATION + "home: " + HOME_PAGE + "\n") + about_indent = False + else: + modified_lines.append(line) + return modified_lines + +def write_to_original(lines: list[str]): + original_recipe = path.abspath(GRAYSKULL_OUTPUT_PATH) + original_meta = path.join(original_recipe, "meta.yaml") + with open(original_meta, "w") as file: + file.writelines(lines) + +def rename_recipe_dir(): + new_recipe_name = path.abspath(path.join(GRAYSKULL_OUTPUT_PATH.replace(".", "-").lower())) + shutil.rmtree(new_recipe_name, ignore_errors=True) + os.replace(path.abspath(GRAYSKULL_OUTPUT_PATH), new_recipe_name) + +if __name__ == "__main__": + original_grayskull_out = read_grayskull_output() + modified_recipe_meta = None + modified_recipe_meta = update_naming_scheme(original_grayskull_out) + modified_recipe_meta = inject_run_exports(modified_recipe_meta) + modified_recipe_meta = suffix_license(modified_recipe_meta) + modified_recipe_meta = inject_home_page(modified_recipe_meta) + write_to_original(modified_recipe_meta) + rename_recipe_dir() \ No newline at end of file