From 0f85bb6e32a0a3ec712450ee58754107f16a698d Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 16 Mar 2022 09:39:14 +0100 Subject: [PATCH 1/3] pre-download model weights in CI docs build --- .circleci/config.yml | 31 +++++++++++++++++++++++-------- scripts/collect_model_urls.py | 18 ++++++++---------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3de0894304b..8fff25d31bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -158,6 +158,26 @@ commands: args: iopath git+https://github.com/pytorch/data descr: Install prototype dependencies + download_model_weights: + parameters: + extract_roots: + type: string + default: "torchvision/models" + background: + type: boolean + default: true + steps: + - apt_install: + args: parallel wget + descr: Install download utilitites + - run: + name: Download model weights + background: << parameters.background >> + command: | + mkdir -p ~/.cache/torch/hub/checkpoints + python scripts/collect_model_urls.py << parameters.extract_roots >> \ + | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci' + # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. # This command can be used if only a selection of tests need to be run, for ad-hoc files. run_tests_selective: @@ -340,14 +360,8 @@ jobs: resource_class: xlarge steps: - checkout - - run: - name: Download model weights - background: true - command: | - sudo apt update -qy && sudo apt install -qy parallel wget - mkdir -p ~/.cache/torch/hub/checkpoints - python scripts/collect_model_urls.py torchvision/prototype/models \ - | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci' + - download_model_weights: + extract_roots: torchvision/prototype/models - install_torchvision - install_prototype_dependencies - pip_install: @@ -1017,6 +1031,7 @@ jobs: - attach_workspace: at: ~/workspace - checkout + - download_model_weights - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh diff --git a/scripts/collect_model_urls.py b/scripts/collect_model_urls.py index 3554e80b1ed..2acba6cbbda 100644 --- a/scripts/collect_model_urls.py +++ b/scripts/collect_model_urls.py @@ -2,21 +2,19 @@ import re import sys -MODEL_URL_PATTERN = re.compile(r"https://download[.]pytorch[.]org/models/.*?[.]pth") +MODEL_URL_PATTERN = re.compile(r"https://download[.]pytorch[.]org/models/.+?[.]pth") -def main(root): +def main(*roots): model_urls = set() - for path in pathlib.Path(root).glob("**/*"): - if path.name.startswith("_") or not path.suffix == ".py": - continue - - with open(path, "r") as file: - for line in file: - model_urls.update(MODEL_URL_PATTERN.findall(line)) + for root in roots: + for path in pathlib.Path(root).rglob("*.py"): + with open(path, "r") as file: + for line in file: + model_urls.update(MODEL_URL_PATTERN.findall(line)) print("\n".join(sorted(model_urls))) if __name__ == "__main__": - main(sys.argv[1]) + main(*sys.argv[1:]) From c0dcb9093a8eca762750a98a2da518a66262c1fb Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 16 Mar 2022 11:49:34 +0100 Subject: [PATCH 2/3] move changes into template --- .circleci/config.yml | 32 ++++++++++++++++---------------- .circleci/config.yml.in | 31 +++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8fff25d31bd..b3649d4fb86 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -158,6 +158,22 @@ commands: args: iopath git+https://github.com/pytorch/data descr: Install prototype dependencies + # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. + # This command can be used if only a selection of tests need to be run, for ad-hoc files. + run_tests_selective: + parameters: + file_or_dir: + type: string + steps: + - run: + name: Install test utilities + command: pip install --progress-bar=off pytest pytest-mock + - run: + name: Run tests + command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> + - store_test_results: + path: test-results + download_model_weights: parameters: extract_roots: @@ -178,22 +194,6 @@ commands: python scripts/collect_model_urls.py << parameters.extract_roots >> \ | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci' - # Most of the test suite is handled by the `unittest` jobs, with completely different workflow and setup. - # This command can be used if only a selection of tests need to be run, for ad-hoc files. - run_tests_selective: - parameters: - file_or_dir: - type: string - steps: - - run: - name: Install test utilities - command: pip install --progress-bar=off pytest pytest-mock - - run: - name: Run tests - command: pytest --junitxml=test-results/junit.xml -v --durations 20 <> - - store_test_results: - path: test-results - binary_common: &binary_common parameters: # Edit these defaults to do a release diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index e36b368db1c..541c696786c 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -174,6 +174,26 @@ commands: - store_test_results: path: test-results + download_model_weights: + parameters: + extract_roots: + type: string + default: "torchvision/models" + background: + type: boolean + default: true + steps: + - apt_install: + args: parallel wget + descr: Install download utilitites + - run: + name: Download model weights + background: << parameters.background >> + command: | + mkdir -p ~/.cache/torch/hub/checkpoints + python scripts/collect_model_urls.py << parameters.extract_roots >> \ + | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci' + binary_common: &binary_common parameters: # Edit these defaults to do a release @@ -340,14 +360,8 @@ jobs: resource_class: xlarge steps: - checkout - - run: - name: Download model weights - background: true - command: | - sudo apt update -qy && sudo apt install -qy parallel wget - mkdir -p ~/.cache/torch/hub/checkpoints - python scripts/collect_model_urls.py torchvision/prototype/models \ - | parallel -j0 'wget --no-verbose -O ~/.cache/torch/hub/checkpoints/`basename {}` {}\?source=ci' + - download_model_weights: + extract_roots: torchvision/prototype/models - install_torchvision - install_prototype_dependencies - pip_install: @@ -1017,6 +1031,7 @@ jobs: - attach_workspace: at: ~/workspace - checkout + - download_model_weights - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh From a893c714f4129195bca5f05b02db3135ac51f60f Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Wed, 16 Mar 2022 11:50:25 +0100 Subject: [PATCH 3/3] change docs image --- .circleci/config.yml | 2 +- .circleci/config.yml.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3649d4fb86..c68f39642ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1025,7 +1025,7 @@ jobs: build_docs: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: circleci/python:3.7 resource_class: 2xlarge+ steps: - attach_workspace: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 541c696786c..731d0133528 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -1025,7 +1025,7 @@ jobs: build_docs: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: circleci/python:3.7 resource_class: 2xlarge+ steps: - attach_workspace: