Skip to content
Snippets Groups Projects
Commit bf632200 authored by George Marchment's avatar George Marchment
Browse files

update download

parent 53754093
No related branches found
No related tags found
No related merge requests found
import json
import os
with open("./wf_crawl_nextflow.json") as json_file:
crawler = json.load(json_file)
#with open("./wf_crawl_nextflow.json") as json_file:
# crawler = json.load(json_file)
#
#crawler.pop("last_date")
crawler.pop("last_date")
def download(path, crawler):
#These are the licenses that we are keeping
LICENSE = ["Apache License 2.0", "GNU General Public License v3.0", "MIT License"]
#These are the licenses that we are keeping
LICENSE = ["Apache License 2.0", "GNU General Public License v3.0", "MIT License"]
#Code for downloading the files
index = 1
downloaded = 0
for project in crawler:
print(f'* {index}/{len(crawler)}')
if(crawler[project]["license"] in LICENSE):
print(f"Downloading...'{project}'")
downloaded+=1
os.system(f"mkdir -p ")
os.system(f'git clone https://github.com/{project}.git ../../../Workflow-Corpus-Open-License/{project}')
index+=1
print('\n\n')
print(downloaded)
#Code for downloading the files
index = 1
downloaded = 0
for project in crawler:
print(f'* {index}/{len(crawler)}')
if(crawler[project]["license"] in LICENSE):
print(f"Downloading...'{project}'")
downloaded+=1
os.system(f"mkdir -p ")
os.system(f'git clone https://github.com/{project}.git {path}/{project}')
index+=1
print('\n\n')
print(downloaded)
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment