diff --git a/README.md b/README.md index b0cb6a7b7f7824b0245563f8d397b589d41263f1..180c3e0ceb54d5e118e31b9a37551bd8e28963b3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ `git clone --recurse-submodules https://gitlab.liris.cnrs.fr/sharefair/bioflow-insight-study.git` +`git submodule update --recursive --remote` + tets ## Getting started diff --git a/github-crawler b/github-crawler index 537540936dd0a3fc07e5f0143d74fa7cf93735bb..bf6322002948c7b3df3657e0bd0e5fd82e7c8bd5 160000 --- a/github-crawler +++ b/github-crawler @@ -1 +1 @@ -Subproject commit 537540936dd0a3fc07e5f0143d74fa7cf93735bb +Subproject commit bf6322002948c7b3df3657e0bd0e5fd82e7c8bd5 diff --git a/study.ipynb b/study.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..4c49c9bc977fc417636d2d550ab932c3d1bbf073 --- /dev/null +++ b/study.ipynb @@ -0,0 +1,280 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from github_crawler.download_corpus import" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* 1/752\n", + "Downloading...'HadrienG/nextflow_rna'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: opérande manquant\n", + "Saisissez « mkdir --help » pour plus d'informations.\n", + "Clonage dans '../../../Workflow-Corpus-Open-License/HadrienG/nextflow_rna'...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "* 2/752\n", + "Downloading...'IARCbioinfo/CODEX-nf'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: opérande manquant\n", + "Saisissez « mkdir --help » pour plus d'informations.\n", + "Clonage dans '../../../Workflow-Corpus-Open-License/IARCbioinfo/CODEX-nf'...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "* 3/752\n", + "Downloading...'AndersenLab/ril-nf'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: opérande manquant\n", + "Saisissez « mkdir --help » pour plus d'informations.\n", + "Clonage dans '../../../Workflow-Corpus-Open-License/AndersenLab/ril-nf'...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "* 4/752\n", + "Downloading...'robsyme/markerdev'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: opérande manquant\n", + "Saisissez « mkdir --help » pour plus d'informations.\n", + "Clonage dans '../../../Workflow-Corpus-Open-License/robsyme/markerdev'...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "* 5/752\n", + "\n", + "\n", + "\n", + "* 6/752\n", + "Downloading...'ctmrbio/BACTpipe'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: opérande manquant\n", + "Saisissez « mkdir --help » pour plus d'informations.\n", + "Clonage dans '../../../Workflow-Corpus-Open-License/ctmrbio/BACTpipe'...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "* 7/752\n", + "Downloading...'danielecook/nextflow-tutorial'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: opérande manquant\n", + "Saisissez « mkdir --help » pour plus d'informations.\n", + "Clonage dans '../../../Workflow-Corpus-Open-License/danielecook/nextflow-tutorial'...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "* 8/752\n", + "Downloading...'IARCbioinfo/bamsurgeon-nf'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: opérande manquant\n", + "Saisissez « mkdir --help » pour plus d'informations.\n", + "Clonage dans '../../../Workflow-Corpus-Open-License/IARCbioinfo/bamsurgeon-nf'...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "* 9/752\n", + "Downloading...'nextflow-io/cwl2nxf'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "mkdir: opérande manquant\n", + "Saisissez « mkdir --help » pour plus d'informations.\n", + "Clonage dans '../../../Workflow-Corpus-Open-License/nextflow-io/cwl2nxf'...\n" + ] + } + ], + "source": [ + "\n", + "current_path= os.getcwd()\n", + "os.chdir(\"github-crawler/\")\n", + "from download_corpus import *\n", + "os.chdir(current_path)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "download_files = True\n", + "repo_to_download = \"../Workflow-Corpus-Open-Licence/\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "#Read crawler results\n", + "with open(\"./github-crawler/wf_crawl_nextflow.json\") as json_file:\n", + " crawler = json.load(json_file)\n", + "\n", + "_ = crawler.pop(\"last_date\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "if(download_files):\n", + " current_directory = os.getcwd()\n", + " os.chdir(current_directory+'/'+repo_to_download)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'/home/gmarchment/Documents/These/Work/Workflow-Corpus-Open-Licence'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.getcwd()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}