Skip to content
Snippets Groups Projects
Commit f6e05ffb authored by Romain Deville's avatar Romain Deville
Browse files

:green_heart: Fixing CI 

Mainly updating `docs/.gitlab-ci.yml` and `docs/_data/plugins.py` to make the CI working.

Comment formatting test for now.
parent 978b99d2
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 22
− 22
View file @ f6e05ffb
...@@ -165,28 +165,28 @@ variables: ...@@ -165,28 +165,28 @@ variables:
# ============================================================================= # =============================================================================
# Jobs in test stage # Jobs in test stage
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
test_tox_format_python: #test_tox_format_python:
<<: *tag_docker # <<: *tag_docker
<<: *image_docker # <<: *image_docker
<<: *stage_test # <<: *stage_test
<<: *before_script_python_dependencies # <<: *before_script_python_dependencies
script: # script:
# Install python tox # # Install python tox
- pip3 install tox # - pip3 install tox
# Run tox # # Run tox
- tox -e format_python # - tox -e format_python
#
test_tox_format_shell: #test_tox_format_shell:
<<: *tag_docker # <<: *tag_docker
<<: *image_docker # <<: *image_docker
<<: *stage_test # <<: *stage_test
<<: *before_script_python_dependencies # <<: *before_script_python_dependencies
script: # script:
# Install python tox # # Install python tox
- pip3 install tox # - pip3 install tox
# Run tox # # Run tox
- tox -e format_shell # - tox -e format_shell
#
# ***************************************************************************** # *****************************************************************************
# VIM MODELINE # VIM MODELINE
# vim: fdm=indent # vim: fdm=indent
......
docs/.gitlab-ci.yml
+ 40
− 112
View file @ f6e05ffb
--- ---
### BEGIN MKDOCS TEMPLATE ###
### WARNING, DO NOT UPDATE CONTENT BETWEEN MKDOCS TEMPLATE TAG !###
### Modified content will be overwritten when updating.###
# GLOBAL CONFIGURATION # GLOBAL CONFIGURATION
# ============================================================================= # =============================================================================
# YAML Anchors # YAML Anchors
...@@ -64,24 +61,6 @@ variables: ...@@ -64,24 +61,6 @@ variables:
bash bash
git git
gcc gcc
musl-dev
jpeg-dev
zlib-dev
libffi-dev
cairo-dev
pango-dev
gdk-pixbuf
jpeg-dev
zlib-dev
freetype-dev
lcms2-dev
openjpeg-dev
tiff-dev
tk-dev
tcl-dev
harfbuzz-dev
fribidi-dev
chromium
# Create virtual environment # Create virtual environment
- virtualenv .venv - virtualenv .venv
# Activate virtual environment # Activate virtual environment
...@@ -104,7 +83,7 @@ variables: ...@@ -104,7 +83,7 @@ variables:
# Start the SSH agent # Start the SSH agent
- eval $(ssh-agent -s) - eval $(ssh-agent -s)
# Add the private SSH key to be able to connect to the perso LIRIS website # Add the private SSH key to be able to connect to the perso LIRIS website
- echo "${SSH_PRIVATE_KEY}" | tr -d '' | ssh-add - - echo "${SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add -
.before_script_export_variable: &before_script_export_variable .before_script_export_variable: &before_script_export_variable
# Update values of some variables depending on the CI_DEPLOY_TYPE # Update values of some variables depending on the CI_DEPLOY_TYPE
...@@ -166,7 +145,7 @@ variables: ...@@ -166,7 +145,7 @@ variables:
fi fi
;; ;;
*) *)
echo -e "ERROR - Variable \`CI_DEPLOY_TYPE\` is not defined !" echo -e "\e[31mERROR - Variable \`CI_DEPLOY_TYPE\` is not defined !"
return 1 return 1
;; ;;
esac esac
...@@ -196,11 +175,11 @@ variables: ...@@ -196,11 +175,11 @@ variables:
.only_dev: &only_dev .only_dev: &only_dev
only: only:
refs: refs:
- *ref_release - *ref_release
- *ref_feature - *ref_feature
- *ref_hotfix - *ref_hotfix
- *ref_bugfix - *ref_bugfix
- *ref_merge_requests - *ref_merge_requests
# Jobs under only_pre_prod anchor will be run on `develop` (i.e. pre-release) # Jobs under only_pre_prod anchor will be run on `develop` (i.e. pre-release)
# and `master` (release) branch. # and `master` (release) branch.
...@@ -307,31 +286,31 @@ variables: ...@@ -307,31 +286,31 @@ variables:
- | - |
if [[ -z "${SSH_KNOWN_HOSTS}" ]] if [[ -z "${SSH_KNOWN_HOSTS}" ]]
then then
echo -e "ERROR - Variable \`SSH_KNOWN_HOSTS_${CI_DEPLOY_TYPE})\` is not defined !" echo -e "\e[31mERROR - Variable \`SSH_KNOWN_HOSTS_${CI_DEPLOY_TYPE})\` is not defined !"
export ERROR="true" export ERROR="true"
fi fi
- | - |
if [[ -z "${SSH_PRIVATE_KEY}" ]] if [[ -z "${SSH_PRIVATE_KEY}" ]]
then then
echo -e "ERROR - Variable \`SSH_PRIVATE_KEY_${CI_DEPLOY_TYPE}\` is not defined !" echo -e "\e[31mERROR - Variable \`SSH_PRIVATE_KEY_${CI_DEPLOY_TYPE}\` is not defined !"
export ERROR="true" export ERROR="true"
fi fi
- | - |
if [[ -z "${ONLINE_DEST}" ]] if [[ -z "${ONLINE_DEST}" ]]
then then
echo -e "ERROR - Variable \`ONLINE_DEST_${CI_DEPLOY_TYPE}\` is not defined !" echo -e "\e[31mERROR - Variable \`ONLINE_DEST_${CI_DEPLOY_TYPE}\` is not defined !"
export ERROR="true" export ERROR="true"
fi fi
- | - |
if [[ -z "${RSYNC_DEST}" ]] if [[ -z "${RSYNC_DEST}" ]]
then then
echo -e "ERROR - Variable \`RSYNC_DEST_${CI_DEPLOY_TYPE}\` is not defined !" echo -e "\e[31mERROR - Variable \`RSYNC_DEST_${CI_DEPLOY_TYPE}\` is not defined !"
export ERROR="true" export ERROR="true"
fi fi
- | - |
if [[ "${ERROR}" == "true" ]] if [[ "${ERROR}" == "true" ]]
then then
echo -e "ERROR - At least one required variable is not defined !" echo -e "\e[31mERROR - At least one required variable is not defined !"
return 1 return 1
fi fi
...@@ -369,11 +348,14 @@ test_build_local: ...@@ -369,11 +348,14 @@ test_build_local:
# Install python test requirements # Install python test requirements
- pip3 install -r requirements.docs.txt - pip3 install -r requirements.docs.txt
# Build local documentation # Build local documentation
- mkdocs build -f mkdocs.local.yml -d site_local - mkdocs build -f mkdocs.local.yml -d site_local
# If everything went right, remove build site # If everything went right, remove build site
- rm -rf site_local - rm -rf site_local
rules:
- exists:
- mkdocs.local.yaml
test_build_monorepo: test_build:
<<: *tag_docker <<: *tag_docker
<<: *image_docker <<: *image_docker
<<: *cache_python <<: *cache_python
...@@ -384,7 +366,7 @@ test_build_monorepo: ...@@ -384,7 +366,7 @@ test_build_monorepo:
# Install python test requirements # Install python test requirements
- pip3 install -r requirements.docs.txt - pip3 install -r requirements.docs.txt
# Build local documentation # Build local documentation
- mkdocs build -d site_monorepo - mkdocs build -d site_monorepo
# If everything went right, remove build site # If everything went right, remove build site
- rm -rf site_monorepo - rm -rf site_monorepo
...@@ -406,25 +388,6 @@ build_html: ...@@ -406,25 +388,6 @@ build_html:
paths: paths:
- site/ - site/
build_pdf:
before_script:
- *before_script_python_dependencies
<<: *tag_docker
<<: *image_docker
<<: *cache_python_pull
<<: *stage_build
script:
# Install python documentations requirements
- pip3 install -r requirements.docs.txt
# Export a variable to build the PDF of the documentation
- export ENABLE_PDF_EXPORT=1
# Build the documentation
- mkdocs build
- cp docs/versions.json site/
artifacts:
paths:
- docs/docs.pdf
# Jobs in deploy stage # Jobs in deploy stage
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Template jobs script deploy previously built html documentation. # Template jobs script deploy previously built html documentation.
...@@ -445,25 +408,29 @@ build_pdf: ...@@ -445,25 +408,29 @@ build_pdf:
- export RSYNC_PATH="/${PROJECT_PATH}${LAST_TAG}" - export RSYNC_PATH="/${PROJECT_PATH}${LAST_TAG}"
- export ONLINE_PATH="${ONLINE_DEST}${PROJECT_PATH}${LAST_TAG}" - export ONLINE_PATH="${ONLINE_DEST}${PROJECT_PATH}${LAST_TAG}"
- mkdir -p "tmp/${RSYNC_PATH}" - mkdir -p "tmp/${RSYNC_PATH}"
- cp site/versions.json versions.json - |
if [[ -f "site/versions.json" ]]
then
cp site/versions.json versions.json
fi
- mv site/* "tmp/${RSYNC_PATH}/" - mv site/* "tmp/${RSYNC_PATH}/"
- rsync -avz "tmp/" "${RSYNC_DEST}" - rsync -avz "tmp/" "${RSYNC_DEST}"
- echo -e " - echo -e "
<!DOCTYPE html> <!DOCTYPE html>\n
<html> <html>\n
<head> <head>\n
<meta charset='utf-8'> <meta charset='utf-8'>\n
<title>Redirecting</title> <title>Redirecting</title>\n
<noscript> <noscript>\n
<meta http-equiv='refresh' content='1; url=${ONLINE_PATH}' /> <meta http-equiv='refresh' content='1; url=${ONLINE_PATH}' />\n
</noscript> </noscript>\n
<script> <script>\n
window.location.replace('${ONLINE_PATH}'); window.location.replace('${ONLINE_PATH}');\n
</script> </script>\n
</head> </head>\n
<body> <body>\n
Redirecting to <a href='${ONLINE_PATH}'>${ONLINE_PATH}</a>... Redirecting to <a href='${ONLINE_PATH}'>${ONLINE_PATH}</a>...\n
</body> </body>\n
</html>" > index.html </html>" > index.html
- rsync -avz index.html "${RSYNC_DEST}${PROJECT_PATH}/" - rsync -avz index.html "${RSYNC_DEST}${PROJECT_PATH}/"
- rsync -avz versions.json "${RSYNC_DEST}${PROJECT_PATH}/" - rsync -avz versions.json "${RSYNC_DEST}${PROJECT_PATH}/"
...@@ -472,7 +439,7 @@ deploy_html_dev: ...@@ -472,7 +439,7 @@ deploy_html_dev:
<<: *script_deploy_html <<: *script_deploy_html
<<: *only_dev <<: *only_dev
variables: variables:
CI_DEPLOY_TYPE: "PRE_PROD" CI_DEPLOY_TYPE: "DEV"
PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/ PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
deploy_html_pre_prod: deploy_html_pre_prod:
...@@ -489,43 +456,6 @@ deploy_html_prod: ...@@ -489,43 +456,6 @@ deploy_html_prod:
CI_DEPLOY_TYPE: "PROD" CI_DEPLOY_TYPE: "PROD"
PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/ PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
.script_deploy_pdf: &script_deploy_pdf
<<: *tag_docker
<<: *image_docker
<<: *cache_python
<<: *stage_deploy
needs:
- build_pdf
before_script:
- *before_script_export_variable
- *before_script_prepare_deployment
script:
- mkdir -p "tmp/${CI_PROJECT_PATH}"
- mv "docs/docs.pdf" "tmp/${CI_PROJECT_PATH}/${CI_PROJECT_NAME}.pdf"
- rsync -avz "tmp/" "${RSYNC_DEST}"
deploy_pdf_dev:
<<: *script_deploy_pdf
<<: *only_dev
variables:
CI_DEPLOY_TYPE: "DEV"
PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
deploy_pdf_pre_prod:
<<: *script_deploy_pdf
<<: *only_pre_prod
variables:
CI_DEPLOY_TYPE: "PRE_PROD"
PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
deploy_pdf_prod:
<<: *stage_deploy
<<: *script_deploy_pdf
<<: *only_prod
variables:
CI_DEPLOY_TYPE: "PROD"
PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
# Jobs in post_deploy stage # Jobs in post_deploy stage
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
trigger_main_repo: trigger_main_repo:
...@@ -555,7 +485,7 @@ inform_triggers_variables: ...@@ -555,7 +485,7 @@ inform_triggers_variables:
- | - |
if [[ -z "${MAIN_PROJECT}" ]] if [[ -z "${MAIN_PROJECT}" ]]
then then
echo -e " echo -e "\e[32m
INFO - To trigger a main repo documentation CI, you can set \n INFO - To trigger a main repo documentation CI, you can set \n
INFO - variable \`MAIN_PROJECT\`. Otherwise, you have nothing\n INFO - variable \`MAIN_PROJECT\`. Otherwise, you have nothing\n
INFO - to do. If the CI you want to run is not at the root of\n INFO - to do. If the CI you want to run is not at the root of\n
...@@ -565,8 +495,6 @@ inform_triggers_variables: ...@@ -565,8 +495,6 @@ inform_triggers_variables:
INFO - the \`MAIN_PROJECT\`." INFO - the \`MAIN_PROJECT\`."
fi fi
### END MKDOCS TEMPLATE ###
# ***************************************************************************** # *****************************************************************************
# VIM MODELINE # VIM MODELINE
# vim: fdm=indent # vim: fdm=indent
......
docs/_data/plugins.py
+ 67
− 15
View file @ f6e05ffb
...@@ -447,25 +447,48 @@ def load_yaml_file(path: str, filename: str) -> None: ...@@ -447,25 +447,48 @@ def load_yaml_file(path: str, filename: str) -> None:
return schema.source, data_type return schema.source, data_type
def update_subrepo_logo_src(env:dict,curr_repo:dict,repo_name:str,subrepo_dict:dict, path:str,external:bool) -> None: def update_subrepo_logo_src(
env: dict,
curr_repo: dict,
repo_name: str,
subrepo_dict: dict,
path: str,
external: bool,
latest: str
) -> None:
"""
@rdeville: TODO
"""
logo_subpath = "" logo_subpath = ""
src_subpath = "" src_subpath = ""
if external: if external:
logo_subpath = os.path.join(subrepo_dict["online_url"]) logo_subpath = os.path.join(subrepo_dict["online_url"])
src_subpath = os.path.join(path.replace(f"{env.project_dir}/",""),repo_name) if latest:
logo_subpath = os.path.join(logo_subpath,latest)
src_subpath = os.path.join(
path.replace(f"{env.project_dir}/", ""), repo_name
)
if "logo" not in curr_repo: if "logo" not in curr_repo:
curr_repo["logo"] = os.path.join(logo_subpath, "assets", "img", "meta",f"{repo_name}_logo.png") curr_repo["logo"] = os.path.join(
logo_subpath, "assets", "img", "meta", f"{repo_name}_logo.png"
)
if "src_path" in curr_repo: if "src_path" in curr_repo:
for i_src in curr_repo["src_path"]: for i_src in curr_repo["src_path"]:
i_src = os.path.join(src_subpath, i_src) i_src = os.path.join(src_subpath, i_src)
env.conf["plugins"]["mkdocstrings"].config.data["handlers"][ env.conf["plugins"]["mkdocstrings"].config.data["handlers"][
"python" "python"
]["setup_commands"].append(f"sys.path.append('{i_src}')") ]["setup_commands"].append(f"sys.path.append('{i_src}')")
print(yaml.dump(curr_repo))
def update_subrepo_info(env: dict, subrepo_list: dict, path: str, external:bool = False) -> dict: def update_subrepo_info(
env: dict, subrepo_list: dict, path: str, external: bool = False
) -> dict:
""" """
@rdeville TODO @rdeville TODO
""" """
...@@ -483,7 +506,24 @@ def update_subrepo_info(env: dict, subrepo_list: dict, path: str, external:bool ...@@ -483,7 +506,24 @@ def update_subrepo_info(env: dict, subrepo_list: dict, path: str, external:bool
print( print(
f"{INFO_CLR}INFO [macros] - Cloning repo {i_repo['name']}{RESET_CLR}" f"{INFO_CLR}INFO [macros] - Cloning repo {i_repo['name']}{RESET_CLR}"
) )
git.Repo.clone_from(i_repo["git_url"], subrepo_root) git_subrepo = git.Repo.clone_from(i_repo["git_url"], subrepo_root)
latest = ""
if git_subrepo.tags:
last_major = 0
last_minor = 0
for i_tag in git_subrepo.tags:
i_tag = yaml.dump(i_tag.path)
i_tag = re.sub(".*v", "", i_tag).split(".")
major = int(i_tag[0])
minor = int(i_tag[1])
if major > last_major:
last_major = major
last_minor = 0
if minor > last_minor:
last_minor = minor
last_patch = 0
latest = f"{last_major}.{last_minor}"
if "subpath" in i_repo: if "subpath" in i_repo:
data_dir = os.path.join( data_dir = os.path.join(
...@@ -496,12 +536,16 @@ def update_subrepo_info(env: dict, subrepo_list: dict, path: str, external:bool ...@@ -496,12 +536,16 @@ def update_subrepo_info(env: dict, subrepo_list: dict, path: str, external:bool
data, _ = load_yaml_file(data_dir, data_file) data, _ = load_yaml_file(data_dir, data_file)
for i_repo_info in data: for i_repo_info in data:
curr_repo = data[i_repo_info] curr_repo = data[i_repo_info]
update_subrepo_logo_src(env,curr_repo,i_repo_info,i_repo,path,external) update_subrepo_logo_src(
env, curr_repo, i_repo_info, i_repo, path, external, latest
)
return_dict.update(data) return_dict.update(data)
return return_dict return return_dict
def update_subrepo(env: dict, subrepo_dict: dict, path: str, external:bool) -> dict: def update_subrepo(
env: dict, subrepo_dict: dict, path: str, external: bool
) -> dict:
""" """
@rdeville TODO @rdeville TODO
""" """
...@@ -513,24 +557,31 @@ def update_subrepo(env: dict, subrepo_dict: dict, path: str, external:bool) -> d ...@@ -513,24 +557,31 @@ def update_subrepo(env: dict, subrepo_dict: dict, path: str, external:bool) -> d
elif i_key == "internal": elif i_key == "internal":
env.variables["internal_subdoc"] = True env.variables["internal_subdoc"] = True
return_dict.update( return_dict.update(
update_subrepo_info(env, subrepo_dict[i_key], path,external) update_subrepo_info(env, subrepo_dict[i_key], path, external)
) )
elif i_key not in ["nav_entry"]: elif i_key not in ["nav_entry"]:
return_dict.update( return_dict.update(
update_subrepo( update_subrepo(
env, subrepo_dict[i_key], os.path.join(path, i_key),external env,
subrepo_dict[i_key],
os.path.join(path, i_key),
external,
) )
) )
return return_dict return return_dict
def update_logo_src_repo(env:dict,curr_repo:dict,repo_name:str,path:str=None) -> None: def update_logo_src_repo(
env: dict, curr_repo: dict, repo_name: str, path: str = None
) -> None:
subpath = "" subpath = ""
if path: if path:
subpath = os.path.join(path.replace(env.project_dir,""),repo_name) subpath = os.path.join(path.replace(env.project_dir, ""), repo_name)
if "logo" not in curr_repo: if "logo" not in curr_repo:
curr_repo["logo"] = os.path.join(subpath, "assets", "img", "meta",f"{repo_name}_logo.png") curr_repo["logo"] = os.path.join(
subpath, "assets", "img", "meta", f"{repo_name}_logo.png"
)
if "src_path" in curr_repo: if "src_path" in curr_repo:
for i_src in curr_repo["src_path"]: for i_src in curr_repo["src_path"]:
i_src = os.path.join(subpath, i_src) i_src = os.path.join(subpath, i_src)
...@@ -539,7 +590,6 @@ def update_logo_src_repo(env:dict,curr_repo:dict,repo_name:str,path:str=None) -> ...@@ -539,7 +590,6 @@ def update_logo_src_repo(env:dict,curr_repo:dict,repo_name:str,path:str=None) ->
]["setup_commands"].append(f"sys.path.append('{i_src}')") ]["setup_commands"].append(f"sys.path.append('{i_src}')")
def load_var_file(env: dict) -> None: def load_var_file(env: dict) -> None:
"""Load variables files in docs/_data/ and variable of subrepo """Load variables files in docs/_data/ and variable of subrepo
...@@ -556,7 +606,7 @@ def load_var_file(env: dict) -> None: ...@@ -556,7 +606,7 @@ def load_var_file(env: dict) -> None:
data, data_type = load_yaml_file(var_dir, i_file) data, data_type = load_yaml_file(var_dir, i_file)
for i_key in data: for i_key in data:
if data_type == "repo": if data_type == "repo":
update_logo_src_repo(env,data[i_key],i_key) update_logo_src_repo(env, data[i_key], i_key)
env.variables[i_key] = data[i_key] env.variables[i_key] = data[i_key]
...@@ -644,7 +694,9 @@ def define_env(env: dict) -> None: ...@@ -644,7 +694,9 @@ def define_env(env: dict) -> None:
if "subrepo" in env.variables: if "subrepo" in env.variables:
env.variables["internal_subdoc"] = False env.variables["internal_subdoc"] = False
env.variables.update( env.variables.update(
update_subrepo(env, env.variables["subrepo"], env.project_dir, False) update_subrepo(
env, env.variables["subrepo"], env.project_dir, False
)
) )
set_config(env) set_config(env)
......
mkdocs.yml
+ 4
− 1
View file @ f6e05ffb
...@@ -28,7 +28,7 @@ theme: ...@@ -28,7 +28,7 @@ theme:
# https://squidfunk.github.io/mkdocs-material/ # https://squidfunk.github.io/mkdocs-material/
name: material name: material
# Overriding parent theme configuration # Overriding parent theme configuration
custom_dir: docs/theme #custom_dir: docs/theme
# Determines whether the search plugin expects the theme to provide a # Determines whether the search plugin expects the theme to provide a
# dedicated search page via a template located at search/search.html. # dedicated search page via a template located at search/search.html.
include_search_page: false include_search_page: false
...@@ -190,6 +190,9 @@ plugins: ...@@ -190,6 +190,9 @@ plugins:
# DO NOT FORGET TO ADD/UPDATE THE \`nav\` AND \`extra\` KEYS BELOW. # DO NOT FORGET TO ADD/UPDATE THE \`nav\` AND \`extra\` KEYS BELOW.
### END MKDOCS TEMPLATE ### ### END MKDOCS TEMPLATE ###
extra:
version:
provider: mike
# Navigation Pane # Navigation Pane
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
nav: nav:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment