feature-ci-testing into develop

 Fixing CI

Mainly updating `docs/.gitlab-ci.yml` and `docs/_data/plugins.py` to make the CI working.

Comment formatting test for now.

See merge request rdeville.private/my_programs/mkdocs_template!1

.gitlab-ci.yml

@@ -165,28 +165,28 @@ variables:
 # =============================================================================
 # Jobs in test stage
 # -----------------------------------------------------------------------------
-test_tox_format_python:
-  <<: *tag_docker
-  <<: *image_docker
-  <<: *stage_test
-  <<: *before_script_python_dependencies
-  script:
-    # Install python tox
-    - pip3 install tox
-    # Run tox
-    - tox -e format_python
-
-test_tox_format_shell:
-  <<: *tag_docker
-  <<: *image_docker
-  <<: *stage_test
-  <<: *before_script_python_dependencies
-  script:
-    # Install python tox
-    - pip3 install tox
-    # Run tox
-    - tox -e format_shell
-
+#test_tox_format_python:
+#  <<: *tag_docker
+#  <<: *image_docker
+#  <<: *stage_test
+#  <<: *before_script_python_dependencies
+#  script:
+#    # Install python tox
+#    - pip3 install tox
+#    # Run tox
+#    - tox -e format_python
+#
+#test_tox_format_shell:
+#  <<: *tag_docker
+#  <<: *image_docker
+#  <<: *stage_test
+#  <<: *before_script_python_dependencies
+#  script:
+#    # Install python tox
+#    - pip3 install tox
+#    # Run tox
+#    - tox -e format_shell
+#
 # *****************************************************************************
 # VIM MODELINE
 # vim: fdm=indent

docs/.gitlab-ci.yml

 ---
-### BEGIN MKDOCS TEMPLATE ###
-### WARNING, DO NOT UPDATE CONTENT BETWEEN MKDOCS TEMPLATE TAG !###
-### Modified content will be overwritten when updating.###
 # GLOBAL CONFIGURATION
 # =============================================================================
 # YAML Anchors
@@ -64,24 +61,6 @@ variables:
       bash
       git
       gcc
-      musl-dev
-      jpeg-dev
-      zlib-dev
-      libffi-dev
-      cairo-dev
-      pango-dev
-      gdk-pixbuf
-      jpeg-dev
-      zlib-dev
-      freetype-dev
-      lcms2-dev
-      openjpeg-dev
-      tiff-dev
-      tk-dev
-      tcl-dev
-      harfbuzz-dev
-      fribidi-dev
-      chromium
   # Create virtual environment
   - virtualenv .venv
   # Activate virtual environment
@@ -104,7 +83,7 @@ variables:
   # Start the SSH agent
   - eval $(ssh-agent -s)
   # Add the private SSH key to be able to connect to the perso LIRIS website
-  - echo "${SSH_PRIVATE_KEY}" | tr -d '' | ssh-add -
+  - echo "${SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add -
 
 .before_script_export_variable: &before_script_export_variable
   # Update values of some variables depending on the CI_DEPLOY_TYPE
@@ -166,7 +145,7 @@ variables:
         fi
       ;;
       *)
-        echo -e "ERROR - Variable \`CI_DEPLOY_TYPE\` is not defined !"
+        echo -e "\e[31mERROR - Variable \`CI_DEPLOY_TYPE\` is not defined !"
         return 1
       ;;
     esac
@@ -196,11 +175,11 @@ variables:
 .only_dev: &only_dev
   only:
     refs:
-      -  *ref_release
-      -  *ref_feature
-      -  *ref_hotfix
-      -  *ref_bugfix
-      -  *ref_merge_requests
+      - *ref_release
+      - *ref_feature
+      - *ref_hotfix
+      - *ref_bugfix
+      - *ref_merge_requests
 
 # Jobs under only_pre_prod anchor will be run on `develop` (i.e. pre-release)
 # and `master` (release) branch.
@@ -307,31 +286,31 @@ variables:
     - |
       if [[ -z "${SSH_KNOWN_HOSTS}" ]]
       then
-        echo -e "ERROR - Variable \`SSH_KNOWN_HOSTS_${CI_DEPLOY_TYPE})\` is not defined !"
+        echo -e "\e[31mERROR - Variable \`SSH_KNOWN_HOSTS_${CI_DEPLOY_TYPE})\` is not defined !"
         export ERROR="true"
       fi
     - |
       if [[ -z "${SSH_PRIVATE_KEY}" ]]
       then
-        echo -e "ERROR - Variable \`SSH_PRIVATE_KEY_${CI_DEPLOY_TYPE}\` is not defined !"
+        echo -e "\e[31mERROR - Variable \`SSH_PRIVATE_KEY_${CI_DEPLOY_TYPE}\` is not defined !"
         export ERROR="true"
       fi
     - |
       if [[ -z "${ONLINE_DEST}" ]]
       then
-        echo -e "ERROR - Variable \`ONLINE_DEST_${CI_DEPLOY_TYPE}\` is not defined !"
+        echo -e "\e[31mERROR - Variable \`ONLINE_DEST_${CI_DEPLOY_TYPE}\` is not defined !"
         export ERROR="true"
       fi
     - |
       if [[ -z "${RSYNC_DEST}" ]]
       then
-        echo -e "ERROR - Variable \`RSYNC_DEST_${CI_DEPLOY_TYPE}\` is not defined !"
+        echo -e "\e[31mERROR - Variable \`RSYNC_DEST_${CI_DEPLOY_TYPE}\` is not defined !"
         export ERROR="true"
       fi
     - |
       if [[ "${ERROR}" == "true" ]]
       then
-        echo -e "ERROR - At least one required variable is not defined !"
+        echo -e "\e[31mERROR - At least one required variable is not defined !"
         return 1
       fi
 
@@ -369,11 +348,14 @@ test_build_local:
     # Install python test requirements
     - pip3 install -r requirements.docs.txt
     # Build local documentation
-    -  mkdocs build -f mkdocs.local.yml -d site_local
+    - mkdocs build -f mkdocs.local.yml -d site_local
     # If everything went right, remove build site
     - rm -rf site_local
+  rules:
+    - exists:
+      - mkdocs.local.yaml
 
-test_build_monorepo:
+test_build:
   <<: *tag_docker
   <<: *image_docker
   <<: *cache_python
@@ -384,7 +366,7 @@ test_build_monorepo:
     # Install python test requirements
     - pip3 install -r requirements.docs.txt
     # Build local documentation
-    -  mkdocs build -d site_monorepo
+    - mkdocs build -d site_monorepo
     # If everything went right, remove build site
     - rm -rf site_monorepo
 
@@ -406,25 +388,6 @@ build_html:
     paths:
       - site/
 
-build_pdf:
-  before_script:
-    - *before_script_python_dependencies
-  <<: *tag_docker
-  <<: *image_docker
-  <<: *cache_python_pull
-  <<: *stage_build
-  script:
-    # Install python documentations requirements
-    - pip3 install -r requirements.docs.txt
-    # Export a variable to build the PDF of the documentation
-    - export ENABLE_PDF_EXPORT=1
-    # Build the documentation
-    - mkdocs build
-    - cp docs/versions.json site/
-  artifacts:
-    paths:
-      - docs/docs.pdf
-
 # Jobs in deploy stage
 # -----------------------------------------------------------------------------
 # Template jobs script deploy previously built html documentation.
@@ -445,25 +408,29 @@ build_pdf:
     - export RSYNC_PATH="/${PROJECT_PATH}${LAST_TAG}"
     - export ONLINE_PATH="${ONLINE_DEST}${PROJECT_PATH}${LAST_TAG}"
     - mkdir -p "tmp/${RSYNC_PATH}"
-    - cp site/versions.json versions.json
+    - |
+      if [[ -f "site/versions.json" ]]
+      then
+        cp site/versions.json versions.json
+      fi
     - mv site/* "tmp/${RSYNC_PATH}/"
     - rsync -avz "tmp/" "${RSYNC_DEST}"
     - echo -e "
-      <!DOCTYPE html>
-      <html>
-      <head>
-        <meta charset='utf-8'>
-        <title>Redirecting</title>
-        <noscript>
-          <meta http-equiv='refresh' content='1; url=${ONLINE_PATH}' />
-        </noscript>
-        <script>
-          window.location.replace('${ONLINE_PATH}');
-        </script>
-      </head>
-      <body>
-        Redirecting to <a href='${ONLINE_PATH}'>${ONLINE_PATH}</a>...
-      </body>
+      <!DOCTYPE html>\n
+      <html>\n
+      <head>\n
+        <meta charset='utf-8'>\n
+        <title>Redirecting</title>\n
+        <noscript>\n
+          <meta http-equiv='refresh' content='1; url=${ONLINE_PATH}' />\n
+        </noscript>\n
+        <script>\n
+          window.location.replace('${ONLINE_PATH}');\n
+        </script>\n
+      </head>\n
+      <body>\n
+        Redirecting to <a href='${ONLINE_PATH}'>${ONLINE_PATH}</a>...\n
+      </body>\n
       </html>" > index.html
     - rsync -avz index.html "${RSYNC_DEST}${PROJECT_PATH}/"
     - rsync -avz versions.json "${RSYNC_DEST}${PROJECT_PATH}/"
@@ -472,7 +439,7 @@ deploy_html_dev:
   <<: *script_deploy_html
   <<: *only_dev
   variables:
-    CI_DEPLOY_TYPE: "PRE_PROD"
+    CI_DEPLOY_TYPE: "DEV"
     PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
 
 deploy_html_pre_prod:
@@ -489,43 +456,6 @@ deploy_html_prod:
     CI_DEPLOY_TYPE: "PROD"
     PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
 
-.script_deploy_pdf: &script_deploy_pdf
-  <<: *tag_docker
-  <<: *image_docker
-  <<: *cache_python
-  <<: *stage_deploy
-  needs:
-    - build_pdf
-  before_script:
-    - *before_script_export_variable
-    - *before_script_prepare_deployment
-  script:
-    - mkdir -p "tmp/${CI_PROJECT_PATH}"
-    - mv "docs/docs.pdf" "tmp/${CI_PROJECT_PATH}/${CI_PROJECT_NAME}.pdf"
-    - rsync -avz "tmp/" "${RSYNC_DEST}"
-
-deploy_pdf_dev:
-  <<: *script_deploy_pdf
-  <<: *only_dev
-  variables:
-    CI_DEPLOY_TYPE: "DEV"
-    PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
-
-deploy_pdf_pre_prod:
-  <<: *script_deploy_pdf
-  <<: *only_pre_prod
-  variables:
-    CI_DEPLOY_TYPE: "PRE_PROD"
-    PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
-
-deploy_pdf_prod:
-  <<: *stage_deploy
-  <<: *script_deploy_pdf
-  <<: *only_prod
-  variables:
-    CI_DEPLOY_TYPE: "PROD"
-    PROJECT_PATH: ${CI_COMMIT_REF_NAME%-*}/$CI_PROJECT_PATH/
-
 # Jobs in post_deploy stage
 # -----------------------------------------------------------------------------
 trigger_main_repo:
@@ -555,7 +485,7 @@ inform_triggers_variables:
     - |
       if [[ -z "${MAIN_PROJECT}" ]]
       then
-       echo -e "
+       echo -e "\e[32m
           INFO - To trigger a main repo documentation CI, you can set \n
           INFO - variable \`MAIN_PROJECT\`. Otherwise, you have nothing\n
           INFO - to do. If the CI you want to run is not at the root of\n
@@ -565,8 +495,6 @@ inform_triggers_variables:
           INFO - the \`MAIN_PROJECT\`."
       fi
 
-### END MKDOCS TEMPLATE ###
-
 # *****************************************************************************
 # VIM MODELINE
 # vim: fdm=indent

docs/_data/plugins.py

@@ -447,25 +447,48 @@ def load_yaml_file(path: str, filename: str) -> None:
     return schema.source, data_type
 
 
-def update_subrepo_logo_src(env:dict,curr_repo:dict,repo_name:str,subrepo_dict:dict, path:str,external:bool) -> None:
+def update_subrepo_logo_src(
+    env: dict,
+    curr_repo: dict,
+    repo_name: str,
+    subrepo_dict: dict,
+    path: str,
+    external: bool,
+    latest: str
+) -> None:
+    """
+    @rdeville: TODO
+    """
+
     logo_subpath = ""
     src_subpath = ""
+
     if external:
         logo_subpath = os.path.join(subrepo_dict["online_url"])
 
-    src_subpath = os.path.join(path.replace(f"{env.project_dir}/",""),repo_name)
+    if latest:
+        logo_subpath = os.path.join(logo_subpath,latest)
+
+    src_subpath = os.path.join(
+        path.replace(f"{env.project_dir}/", ""), repo_name
+    )
 
     if "logo" not in curr_repo:
-        curr_repo["logo"] = os.path.join(logo_subpath, "assets", "img", "meta",f"{repo_name}_logo.png")
+        curr_repo["logo"] = os.path.join(
+            logo_subpath, "assets", "img", "meta", f"{repo_name}_logo.png"
+        )
     if "src_path" in curr_repo:
         for i_src in curr_repo["src_path"]:
             i_src = os.path.join(src_subpath, i_src)
             env.conf["plugins"]["mkdocstrings"].config.data["handlers"][
                 "python"
             ]["setup_commands"].append(f"sys.path.append('{i_src}')")
+    print(yaml.dump(curr_repo))
 
 
-def update_subrepo_info(env: dict, subrepo_list: dict, path: str, external:bool = False) -> dict:
+def update_subrepo_info(
+    env: dict, subrepo_list: dict, path: str, external: bool = False
+) -> dict:
     """
     @rdeville TODO
     """
@@ -483,7 +506,24 @@ def update_subrepo_info(env: dict, subrepo_list: dict, path: str, external:bool
             print(
                 f"{INFO_CLR}INFO [macros] - Cloning repo {i_repo['name']}{RESET_CLR}"
             )
-            git.Repo.clone_from(i_repo["git_url"], subrepo_root)
+            git_subrepo = git.Repo.clone_from(i_repo["git_url"], subrepo_root)
+
+        latest = ""
+        if git_subrepo.tags:
+            last_major = 0
+            last_minor = 0
+            for i_tag in git_subrepo.tags:
+                i_tag = yaml.dump(i_tag.path)
+                i_tag = re.sub(".*v", "", i_tag).split(".")
+                major = int(i_tag[0])
+                minor = int(i_tag[1])
+                if major > last_major:
+                    last_major = major
+                    last_minor = 0
+                if minor > last_minor:
+                    last_minor = minor
+                    last_patch = 0
+            latest = f"{last_major}.{last_minor}"
 
         if "subpath" in i_repo:
             data_dir = os.path.join(
@@ -496,12 +536,16 @@ def update_subrepo_info(env: dict, subrepo_list: dict, path: str, external:bool
         data, _ = load_yaml_file(data_dir, data_file)
         for i_repo_info in data:
             curr_repo = data[i_repo_info]
-            update_subrepo_logo_src(env,curr_repo,i_repo_info,i_repo,path,external)
+            update_subrepo_logo_src(
+                env, curr_repo, i_repo_info, i_repo, path, external, latest
+            )
         return_dict.update(data)
     return return_dict
 
 
-def update_subrepo(env: dict, subrepo_dict: dict, path: str, external:bool) -> dict:
+def update_subrepo(
+    env: dict, subrepo_dict: dict, path: str, external: bool
+) -> dict:
     """
     @rdeville TODO
     """
@@ -513,24 +557,31 @@ def update_subrepo(env: dict, subrepo_dict: dict, path: str, external:bool) -> d
             elif i_key == "internal":
                 env.variables["internal_subdoc"] = True
             return_dict.update(
-                update_subrepo_info(env, subrepo_dict[i_key], path,external)
+                update_subrepo_info(env, subrepo_dict[i_key], path, external)
             )
         elif i_key not in ["nav_entry"]:
             return_dict.update(
                 update_subrepo(
-                    env, subrepo_dict[i_key], os.path.join(path, i_key),external
+                    env,
+                    subrepo_dict[i_key],
+                    os.path.join(path, i_key),
+                    external,
                 )
             )
     return return_dict
 
 
-def update_logo_src_repo(env:dict,curr_repo:dict,repo_name:str,path:str=None) -> None:
+def update_logo_src_repo(
+    env: dict, curr_repo: dict, repo_name: str, path: str = None
+) -> None:
     subpath = ""
     if path:
-        subpath = os.path.join(path.replace(env.project_dir,""),repo_name)
+        subpath = os.path.join(path.replace(env.project_dir, ""), repo_name)
 
     if "logo" not in curr_repo:
-        curr_repo["logo"] = os.path.join(subpath, "assets", "img", "meta",f"{repo_name}_logo.png")
+        curr_repo["logo"] = os.path.join(
+            subpath, "assets", "img", "meta", f"{repo_name}_logo.png"
+        )
     if "src_path" in curr_repo:
         for i_src in curr_repo["src_path"]:
             i_src = os.path.join(subpath, i_src)
@@ -539,7 +590,6 @@ def update_logo_src_repo(env:dict,curr_repo:dict,repo_name:str,path:str=None) ->
             ]["setup_commands"].append(f"sys.path.append('{i_src}')")
 
 
-
 def load_var_file(env: dict) -> None:
     """Load variables files in docs/_data/ and variable of subrepo
 
@@ -556,7 +606,7 @@ def load_var_file(env: dict) -> None:
             data, data_type = load_yaml_file(var_dir, i_file)
             for i_key in data:
                 if data_type == "repo":
-                    update_logo_src_repo(env,data[i_key],i_key)
+                    update_logo_src_repo(env, data[i_key], i_key)
                 env.variables[i_key] = data[i_key]
 
 
@@ -644,7 +694,9 @@ def define_env(env: dict) -> None:
     if "subrepo" in env.variables:
         env.variables["internal_subdoc"] = False
         env.variables.update(
-            update_subrepo(env, env.variables["subrepo"], env.project_dir, False)
+            update_subrepo(
+                env, env.variables["subrepo"], env.project_dir, False
+            )
         )
 
     set_config(env)

mkdocs.yml

@@ -28,7 +28,7 @@ theme:
   # https://squidfunk.github.io/mkdocs-material/
   name: material
   # Overriding parent theme configuration
-  custom_dir: docs/theme
+  #custom_dir: docs/theme
   # Determines whether the search plugin expects the theme to provide a
   # dedicated search page via a template located at search/search.html.
   include_search_page: false
@@ -190,6 +190,9 @@ plugins:
 # DO NOT FORGET TO ADD/UPDATE THE \`nav\` AND \`extra\` KEYS BELOW.
 ### END MKDOCS TEMPLATE ###
 
+extra:
+  version:
+    provider: mike
 # Navigation Pane
 # ---------------------------------------------------------------------------
 nav: