Adapt docgen to use the mkdocs material theme

.ci-dockerfiles/stdlib-builder/Dockerfile

@@ -14,5 +14,4 @@ RUN apk update \
   python3-dev \
   py3-pip \
   && pip3 install --upgrade pip \
-  && pip3 install mkdocs==1.3.1\
-  && pip3 install mkdocs-ponylang
+  && pip3 install mkdocs mkdocs-material

.release-notes/next-release.md

@@ -16,3 +16,12 @@ This could lead to very surprising results if `reserve` had been called by `unde
 
 In order to make documentation on a class, actor, primitive or struct easier to find, they are now sorted alphabetically inside their package.
 
+## Adapt documentation generation to only depend the mkdocs material theme
+
+We used to have our own mkdocs theme for ponylang documentation, it was based off of the mkdocs-material theme. This change puts the adaptations we need into the mkdocs.yml file itself, so we don't depend on our own theme anymore, but only on the upstream mkdocs-material. This eases the maintenance burden by not having to catch up with the material theme codebase.
+
+This is a **Breaking Change** in so far that users that want to generate documentation with ponyc now have to install the python package `mkdocs-material` instead of `mkdocs-ponylang` to generate their documentation in HTML form with mkdocs. The [library documentation github action](https://github.com/ponylang/library-documentation-action) will have the correct dependencies installed, so no changes should be needed when using it.
+
+## Fix broken documentation generation on Windows
+
+Generating documentation has been broken on windows: `ponyc` was not able to write the documentation files, due to path handling issues on windows. This is now fixed and documentation generation is working again on windows.

CHANGELOG.md

@@ -8,13 +8,14 @@ All notable changes to the Pony compiler and standard library will be documented
 
 - Avoid fairly easy to trigger overflow in Windows Time.nanos code ([PR #4227](https://github.com/ponylang/ponyc/pull/4227))
 - Fix incorrect interaction between String/Array reserve and Pointer realloc ([PR #4223](https://github.com/ponylang/ponyc/pull/4223))
+- Fix broken docgen on Windows ([PR #4226](https://github.com/ponylang/ponyc/pull/4226))
 
 ### Added
 
-
 ### Changed
 
 - Sort package types in documentation ([PR #4228](https://github.com/ponylang/ponyc/pull/4228))
+- Adapt docgen to use the mkdocs material theme ([PR #4226](https://github.com/ponylang/ponyc/pull/4226)
 
 ## [0.51.4] - 2022-10-29
 

src/libponyc/pass/docgen.c

@@ -48,9 +48,11 @@ typedef struct docgen_t
   FILE* type_file;
   const char* base_dir;
   const char* sub_dir;
+  const char* assets_dir;
   char* doc_source_dir;
   size_t base_dir_buf_len;
   size_t sub_dir_buf_len;
+  size_t assets_dir_buf_len;
   errors_t* errors;
   doc_sources_t* included_sources; // As a linked list, being the first element or NULL
 } docgen_t;
@@ -254,13 +256,8 @@ static char* write_tqfn(ast_t* type, const char* type_name, size_t* out_size)
   return buffer;
 }
 
-// Open a file with the specified info.
-// The given filename extension should include a dot if one is needed.
-// The returned file handle must be fclosed() with no longer needed.
-// If the specified file cannot be opened an error will be generated and NULL
-// returned.
-static FILE* doc_open_file(docgen_t* docgen, bool in_sub_dir,
-  const char* filename, const char* extn)
+static FILE* doc_open_file_(docgen_t* docgen, const char* dir,
+  const char* filename, const char* extn, bool binary)
 {
   pony_assert(docgen != NULL);
   pony_assert(filename != NULL);
@@ -269,12 +266,12 @@ static FILE* doc_open_file(docgen_t* docgen, bool in_sub_dir,
   // Build the type file name in a buffer.
   // Full file name is:
   //   directory/filenameextn
-  const char* dir = in_sub_dir ? docgen->sub_dir : docgen->base_dir;
   size_t buf_len;
   char* buffer = doc_cat(dir, filename, extn, "", "", &buf_len);
 
   // Now we have the file name open the file
-  FILE* file = fopen(buffer, "w");
+  const char* mode = binary ? "wb" : "w";
+  FILE* file = fopen(buffer, mode);
 
   if(file == NULL)
     errorf(docgen->errors, NULL,
@@ -284,6 +281,19 @@ static FILE* doc_open_file(docgen_t* docgen, bool in_sub_dir,
   return file;
 }
 
+// Open a file with the specified info.
+// The given filename extension should include a dot if one is needed.
+// The returned file handle must be fclosed() with no longer needed.
+// If the specified file cannot be opened an error will be generated and NULL
+// returned.
+static FILE* doc_open_file(docgen_t* docgen, const char* dir, const char* filename, const char* extn) {
+  return doc_open_file_(docgen, dir, filename, extn, false);
+}
+
+static FILE* doc_open_binary_file(docgen_t* docgen, const char* dir, const char* filename, const char* extn) {
+  return doc_open_file_(docgen, dir, filename, extn, true);
+}
+
 // Functions to handle types
 
 static void doc_type_list(docgen_t* docgen, docgen_opt_t* docgen_opt, ast_t* list,
@@ -486,7 +496,7 @@ static void add_source_code_link(docgen_t* docgen, ast_t* elem)
   if (doc_source != NULL) {
       fprintf(
         docgen->type_file,
-        "\n<span class=\"source-link\">[[Source]](%s#L%zd)</span>\n",
+        "\n<span class=\"source-link\">[[Source]](%s#L-0-%zd)</span>\n",
         doc_source->doc_path, ast_line(elem)
       );
   } else {
@@ -836,11 +846,17 @@ static doc_sources_t* copy_source_to_doc_src(docgen_t* docgen, source_t* source,
   FILE* file = fopen(path, "w");
 
   if (file != NULL) {
-
+    // tell the mkdocs theme to hide the right sidebar
+    // so we have all the space we need for long lines
+    fprintf(file, "---\n");
+    fprintf(file, "hide:\n");
+    fprintf(file, "  - toc\n");
+    fprintf(file, "---\n");
     // Escape markdown to tell this is Pony code
     // Using multiple '```````'  so hopefully the markdown parser
     // will consider the whole text as a code block.
-    fprintf(file, "```````pony-full-source\n");
+    // add line nums as well
+    fprintf(file, "```````pony linenums=\"1\"\n");
     fprintf(file, "%s", source->m);
     fprintf(file, "\n```````");
     fclose(file);
@@ -858,10 +874,10 @@ static doc_sources_t* copy_source_to_doc_src(docgen_t* docgen, source_t* source,
 
     return result;
   } else {
+    errorf(docgen->errors, NULL, "Could not write source-file to %s", path);
     ponyint_pool_free_size(filename_alloc_size, (void*) filename);
     ponyint_pool_free_size(doc_path_alloc_size, (void*) doc_path);
     ponyint_pool_free_size(file_path_alloc_size, (void*) path);
-    errorf(docgen->errors, NULL, "Could not write documentation to file %s", filename);
     return NULL;
   }
 }
@@ -956,7 +972,7 @@ static void doc_entity(docgen_t* docgen, docgen_opt_t* docgen_opt, ast_t* ast)
   size_t tqfn_len;
   char* tqfn = write_tqfn(ast, NULL, &tqfn_len);
 
-  docgen->type_file = doc_open_file(docgen, true, tqfn, ".md");
+  docgen->type_file = doc_open_file(docgen, docgen->sub_dir, tqfn, ".md");
 
   if(docgen->type_file == NULL)
     return;
@@ -1094,7 +1110,7 @@ static void doc_package_home(docgen_t* docgen,
   fprintf(docgen->index_file, "- package %s:\n",
     package_qualified_name(package));
 
-  docgen->type_file = doc_open_file(docgen, true, tqfn, ".md");
+  docgen->type_file = doc_open_file(docgen, docgen->sub_dir, tqfn, ".md");
 
   if(docgen->type_file == NULL)
     return;
@@ -1243,7 +1259,10 @@ static void doc_rm_star(const char* path)
   PONY_DIR* dir = pony_opendir(path, &err);
 
   if(dir == NULL)
+  {
+    printf("Couldn't open %s", path);
     return;
+  }
 
   PONY_DIRINFO* result;
 
@@ -1297,16 +1316,30 @@ static void doc_setup_dirs(docgen_t* docgen, ast_t* program, pass_opt_t* opt)
   docgen->sub_dir = doc_cat(docgen->base_dir, "docs/", "", "", "",
     &docgen->sub_dir_buf_len);
 
+  docgen->assets_dir = doc_cat(docgen->sub_dir, PONYLANG_MKDOCS_ASSETS_DIR, "", "", "",
+    &docgen->assets_dir_buf_len);
+
+  docgen->doc_source_dir = (char*) ponyint_pool_alloc_size(FILENAME_MAX);
+  path_cat(docgen->base_dir, "docs/src", docgen->doc_source_dir);
+
   if(opt->verbosity >= VERBOSITY_INFO)
     fprintf(stderr, "Writing docs to %s\n", docgen->base_dir);
 
   // Create and clear out base directory
   pony_mkdir(docgen->base_dir);
   doc_rm_star(docgen->base_dir);
 
+  // Create and clear out source directory
+  pony_mkdir(docgen->doc_source_dir);
+  doc_rm_star(docgen->doc_source_dir);
+
   // Create and clear out sub directory
   pony_mkdir(docgen->sub_dir);
   doc_rm_star(docgen->sub_dir);
+
+  // Create extra css directory (within sub_dir)
+  pony_mkdir(docgen->assets_dir);
+  doc_rm_star(docgen->assets_dir);
 }
 
 void generate_docs(ast_t* program, pass_opt_t* options)
@@ -1325,15 +1358,11 @@ void generate_docs(ast_t* program, pass_opt_t* options)
   doc_setup_dirs(&docgen, program, options);
 
   // Open the index and home files
-  docgen.index_file = doc_open_file(&docgen, false, "mkdocs", ".yml");
-  docgen.home_file = doc_open_file(&docgen, true, "index", ".md");
+  docgen.index_file = doc_open_file(&docgen, docgen.base_dir, "mkdocs", ".yml");
+  docgen.home_file = doc_open_file(&docgen, docgen.sub_dir, "index", ".md");
   docgen.type_file = NULL;
   docgen.included_sources = NULL;
 
-  docgen.doc_source_dir = (char*) ponyint_pool_alloc_size(FILENAME_MAX);
-  path_cat(docgen.base_dir, "docs/src", docgen.doc_source_dir);
-  pony_mkdir(docgen.doc_source_dir);
-
   // Write documentation files
   if(docgen.index_file != NULL && docgen.home_file != NULL)
   {
@@ -1343,20 +1372,42 @@ void generate_docs(ast_t* program, pass_opt_t* options)
     fprintf(docgen.home_file, "Packages\n\n");
 
     fprintf(docgen.index_file, "site_name: %s\n", name);
-    fprintf(docgen.index_file, "theme: ponylang\n");
+    fprintf(docgen.index_file, "theme:\n");
+    fprintf(docgen.index_file, "  name: material\n");
+    fprintf(docgen.index_file, "  logo: %s%s\n", PONYLANG_MKDOCS_ASSETS_DIR, PONYLANG_MKDOCS_LOGO_FILE);
+    fprintf(docgen.index_file, "  favicon: %s%s\n", PONYLANG_MKDOCS_ASSETS_DIR, PONYLANG_MKDOCS_LOGO_FILE);
+    fprintf(docgen.index_file, "  palette:\n");
+    fprintf(docgen.index_file, "    scheme: ponylang\n");
+    fprintf(docgen.index_file, "    primary: brown\n");
+    fprintf(docgen.index_file, "    accent: amber\n");
+    fprintf(docgen.index_file, "  features:\n");
+    fprintf(docgen.index_file, "    - navigation.top\n");
+
+    fprintf(docgen.index_file, "extra_css:\n");
+    fprintf(docgen.index_file, "  - %s%s\n", PONYLANG_MKDOCS_ASSETS_DIR, PONYLANG_MKDOCS_CSS_FILE);
+
     fprintf(docgen.index_file, "markdown_extensions:\n");
-    fprintf(docgen.index_file, "- markdown.extensions.toc:\n");
-    fprintf(docgen.index_file, "    permalink: true\n");
+    fprintf(docgen.index_file, "  - pymdownx.highlight:\n");
+    fprintf(docgen.index_file, "      anchor_linenums: true\n");
+    fprintf(docgen.index_file, "      line_anchors: \"L\"\n");
+    fprintf(docgen.index_file, "      use_pygments: true\n");
+    fprintf(docgen.index_file, "  - pymdownx.superfences\n");
+    fprintf(docgen.index_file, "  - toc:\n");
+    fprintf(docgen.index_file, "      permalink: true\n");
+    fprintf(docgen.index_file, "      toc_depth: 3\n");
+
     fprintf(docgen.index_file, "nav:\n");
     fprintf(docgen.index_file, "- %s: index.md\n", name);
 
     doc_packages(&docgen, &docgen_opt, program);
   }
 
-  if (docgen.included_sources != NULL) {
+  if(docgen.included_sources != NULL)
+  {
     fprintf(docgen.index_file, "- source:\n");
     doc_sources_t* current_source = docgen.included_sources;
-    while (current_source != NULL) {
+    while(current_source != NULL)
+    {
       fprintf(docgen.index_file, "  - %s : \"%s\" \n" , current_source->filename, current_source->doc_path);
       ponyint_pool_free_size(current_source->filename_alloc_size, (void*) current_source->filename);
       ponyint_pool_free_size(current_source->doc_path_alloc_size, (void*) current_source->doc_path);
@@ -1367,6 +1418,31 @@ void generate_docs(ast_t* program, pass_opt_t* options)
     }
   }
 
+  // write extra css file for the theme
+  FILE* css_file = doc_open_file(&docgen, docgen.assets_dir, PONYLANG_MKDOCS_CSS_FILE, "");
+  if(css_file != NULL)
+  {
+    fprintf(css_file, PONYLANG_MKDOCS_CSS);
+    fclose(css_file);
+  }
+  else
+  {
+    printf("Unable to open file %s%s", docgen.assets_dir, PONYLANG_MKDOCS_CSS_FILE);
+  }
+
+  // write logo png
+  FILE* logo_file = doc_open_binary_file(&docgen, docgen.assets_dir, PONYLANG_MKDOCS_LOGO_FILE, "");
+  if(logo_file != NULL)
+  {
+    const unsigned char logo[PONYLANG_LOGO_LEN] = PONYLANG_LOGO;
+    fwrite(logo, sizeof(unsigned char), PONYLANG_LOGO_LEN, logo_file);
+    fclose(logo_file);
+  }
+  else
+  {
+    printf("Unable to open file %s%s", docgen.assets_dir, PONYLANG_MKDOCS_LOGO_FILE);
+  }
+
   // Tidy up
   if(docgen.index_file != NULL)
     fclose(docgen.index_file);
@@ -1379,4 +1455,7 @@ void generate_docs(ast_t* program, pass_opt_t* options)
 
   if(docgen.sub_dir != NULL)
     ponyint_pool_free_size(docgen.sub_dir_buf_len, (void*)docgen.sub_dir);
+
+  if(docgen.assets_dir != NULL)
+    ponyint_pool_free_size(docgen.assets_dir_buf_len, (void*)docgen.assets_dir);
 }

src/libponyc/pass/docgen.h

@@ -4,9 +4,18 @@
 #include <platform.h>
 #include "../ast/ast.h"
 #include "pass.h"
+#include "../logo.h"
 
 PONY_EXTERN_C_BEGIN
 
+#define PONYLANG_MKDOCS_ASSETS_DIR "assets/"
+#define PONYLANG_MKDOCS_CSS_FILE "ponylang.css"
+#define PONYLANG_MKDOCS_LOGO_FILE "logo.png"
+#define PONYLANG_MKDOCS_CSS "[data-md-color-scheme=\"ponylang\"] {\n"\
+                            "  --md-typeset-a-color: var(--md-primary-fg-color);\n"\
+                            "}"
+
+
 // Generate docs for the given program
 void generate_docs(ast_t* program, pass_opt_t* options);
 

src/libponyc/pkg/package.c

@@ -1006,7 +1006,7 @@ ast_t* package_load(ast_t* from, const char* path, pass_opt_t* opt)
     if(from == program)
     {
       // construct the qualified name from the basename of the full path
-      const char* basepath = strrchr(full_path, '/');
+      const char* basepath = strrchr(full_path, PATH_SLASH);
       if(basepath == NULL)
       {
         basepath = full_path;