Build pages according to their dependency trees

1 files changed, 107 insertions, 109 deletions

diff --git a/app/directory.ts b/app/directory.ts
index ee6a5a9..942803a 100644
--- a/app/directory.ts
+++ b/app/directory.ts
@@ -3,6 +3,52 @@ import { readFileSync } from 'fs';
 import glob from 'glob';
 import { logger } from './logger.js'
 
+/**
+ * Build a page.
+ * 
+ * @param path standard name for page
+ * @returns newly built page, or undefined
+ */
+export function buildPage(directory: PageDirectory, page: Page) {
+    const result = parser.parse(directory, page.raw);
+    const title = result.metadata.displayTitle ?? page.standardName;
+    const content = `${result.metadata.notitle ? '' : `<h1>${title}</h1>`}${result.html}`;
+
+    page.html = content;
+    page.buildTime = Date.now();
+    page.metadata.includeInNavbar = result.metadata.primary ?? false;
+    page.metadata.sortOrder = result.metadata.sortOrder ?? -1;
+    page.metadata.showTitle = !result.metadata.notitle ?? true;
+    page.metadata.displayTitle = title;
+}
+
+/**
+ * Convert a page name to a standard name.
+ * A standard name is the key used by the page directory.
+ * 
+ * @param name non-standard name for a page
+ */
+export function convertNameToStandard(name: string): string {
+    name = name.replace(/[^a-z0-9:]/gi, '_').toLowerCase();
+    if (!name.includes(':')) {
+        name = `main:${name}`;
+    }
+    return name;
+}
+
+/**
+ * Convert a standard name to a file path.
+ * 
+ * @param name standard name for a page
+ */
+export function convertStandardToFilePath(name: string): string {
+    const [first, second] = name.replace('main:', '').split(':');
+    const [title, subpage] = ((second) ? second : first).split('.')
+    const namespace = (second) ? first : undefined
+
+    return `${namespace ? `${namespace}/` : ''}${title}${subpage ? `.${subpage}` : ''}.wiki`
+}
+
 export class PageDirectory {
     pages: Record<string, Page>;
     primaryPages: Page[];
@@ -32,15 +78,16 @@ export class PageDirectory {
 
         const pages = glob.sync(`**/*.wiki`, { cwd: this.pagePath })
 
+        // Load page content
         pages.forEach(page => {
-            page = this.convertNameToStandard(page.replace('.wiki', '').replace('/', ':'));
+            page = convertNameToStandard(page.replace('.wiki', '').replace('/', ':'));
             this.pages[page] = {
                 standardName: page,
                 raw: this.loadRaw(page),
                 buildTime: 0,
                 metadata: {
-                    dependencies: [],
-                    dependents: [],
+                    dependencies: new Set(),
+                    dependents: new Set(),
                     errors: []
                 }
             }
@@ -48,12 +95,21 @@ export class PageDirectory {
 
         const dependencyGraph: Record<string, string[]>  = {};
 
-        Object.keys(this.pages).forEach(name => dependencyGraph[name] = Array.from(parser.findDependencies(this.pages[name].raw)).map(e => this.convertNameToStandard(e)));
+        Object.keys(this.pages).forEach(name => dependencyGraph[name] = Array.from(parser.findDependencies(this.pages[name].raw)).map(e => convertNameToStandard(e)));
 
-        function traverse(dependents: string[], dependencies: string[], recursionCount: number) {
+        // Revursive dependency graph traversal function
+        function traverseGraph(dependents: string[], current: string, dependencies: string[], recursionCount: number, pages: Record<string, Page>) {
             if (recursionCount > parseInt(process.env.PARSER_MAX_RECURSION, 10)) {
                 throw new RecursionError('max recursion reached');
             }
+
+            dependencies?.forEach(e => { 
+                pages[current]?.metadata.dependencies.add(e)
+                if (e !== current) {
+                    pages[e]?.metadata.dependents.add(current);
+                }
+            });
+
             dependencies?.forEach((dependency: string) => {
                 if (dependencyGraph[dependency]?.length != 0) {
                     dependents.forEach((dependent: string) => {
@@ -61,49 +117,56 @@ export class PageDirectory {
                             throw new DependencyError(`circular dependency between ${dependent} and ${dependency}`, [dependent, dependency]);
                         }
                     });
-                    traverse([...dependents, dependency], dependencyGraph[dependency], recursionCount + 1);
+                    traverseGraph([...dependents, dependency], dependency, dependencyGraph[dependency], recursionCount + 1, pages);
                 }
             });
         }
 
+        // Catch circular dependencies and build dependency tree
         Object.keys(dependencyGraph).forEach(name => { 
-            dependencyGraph[name].forEach(dependency => {
-                try {
-                    traverse([name, dependency], dependencyGraph[dependency], 1);
-                } catch (e) {
-                    if (e instanceof RecursionError) {
+            try {
+                traverseGraph([name], name, dependencyGraph[name], 1, this.pages);
+            } catch (e) {
+                if (e instanceof RecursionError) {
+                    this.pages[name].metadata.errors.push({
+                        identifier: 'max-recursion-reached', 
+                        message: `maximum dependency depth of ${process.env.PARSER_MAX_RECURSION} reached`
+                    })
+                    logger.warn(`max recursion for ${name} reached`)
+                } else if (e instanceof DependencyError) {
+                    if (e.pages.includes(name)) {
                         this.pages[name].metadata.errors.push({
-                            identifier: 'max-recursion-reached', 
-                            message: `maximum dependency depth of ${process.env.PARSER_MAX_RECURSION} reached`
+                            identifier: 'circular-dependency', 
+                            message: e.message
                         })
-                        logger.warn(`max recursion for ${name} reached`)
-                    } else if (e instanceof DependencyError) {
-                        if (e.pages.includes(name)) {
-                            this.pages[name].metadata.errors.push({
-                                identifier: 'circular-dependency', 
-                                message: e.message
-                            })
-                            logger.warn(`${e.pages[0]} has a circular dependency with ${e.pages[1]}`)
-                        } else {
-                            logger.warn(`transclusions on page ${name} may not resolve due to dependency errors in its dependency tree`)
-                        }
+                        logger.warn(`${e.pages[0]} has a circular dependency with ${e.pages[1]}`)
                     } else {
-                        throw e;
+                        logger.warn(`transclusions on page ${name} may not resolve due to dependency errors in its dependency tree`)
                     }
+                } else {
+                    throw e;
                 }
-            });
+            }
         });
 
+        function recursiveBulld(pages: Record<string, Page>, current: Page, directory: PageDirectory, buildPage: (directory: PageDirectory, page: Page) => void) {
+            if (current.metadata.errors.length == 0) {
+                current.metadata.dependencies.forEach(dependency => {
+                    if (pages[dependency].buildTime == 0) {
+                        recursiveBulld(pages, pages[dependency], directory, buildPage);
+                    }
+                });
+                buildPage(directory, current)
+            }
+        }
+
+        // Build pages in order
         const primaryPages = [];
         Object.keys(this.pages).forEach(name => {
-            if (this.pages[name].metadata.errors.length == 0) {
-                this.pages[name] = this.buildPage(name);
-                if (this.pages[name].metadata.includeInNavbar) {
-                    primaryPages.push(this.pages[name]);
-                }
-            }
+            recursiveBulld(this.pages, this.pages[name], this, buildPage);
         });
 
+        // Sort primary pages
         primaryPages.sort((a, b) => {
             return a.metadata.sortOrder - b.metadata.sortOrder;
         });
@@ -119,7 +182,7 @@ export class PageDirectory {
      * @returns whether the page exists
      */
     exists(name: string): boolean {
-        return !!this.pages[this.convertNameToStandard(name)];
+        return !!this.pages[convertNameToStandard(name)];
     }
 
     /**
@@ -129,7 +192,7 @@ export class PageDirectory {
      * @returns page
      */
     get(name: string): Page {
-        name = this.convertNameToStandard(name);
+        name = convertNameToStandard(name);
         const page = this.pages[name];
         if (!page) {
             return undefined;
@@ -145,7 +208,7 @@ export class PageDirectory {
      * @returns raw wikitext
      */
     getRaw(name: string): string {
-        name = this.convertNameToStandard(name);
+        name = convertNameToStandard(name);
         return this.pages[name]?.raw;
     }
 
@@ -156,16 +219,16 @@ export class PageDirectory {
      * @returns whether the page was rebuilt
      */
     purge(name: string): boolean {
-        name = this.convertNameToStandard(name);
+        name = convertNameToStandard(name);
         const page = this.pages[name];
         if (page) {
             if (page.buildTime + parseInt(process.env.PURGE_COOLDOWN_MIN, 10) * 60 * 1000 > Date.now()) {
                 return false;
             } else {
-                delete this.pages[name];
-                if (this.buildPage(name)) {
-                    return true;
-                }
+                // delete this.pages[name];
+                // if (this.buildPage(name)) {
+                //     return true;
+                // }
             }
         }
         return false;
@@ -191,80 +254,15 @@ export class PageDirectory {
     }
 
     private loadRaw(name: string): string {
-        name = this.convertNameToStandard(name);
+        name = convertNameToStandard(name);
         let data: string;
         try {
-            data = readFileSync(`${this.pagePath}/${this.convertStandardToFilePath(name)}`, 'utf-8'); 
+            data = readFileSync(`${this.pagePath}/${convertStandardToFilePath(name)}`, 'utf-8'); 
         } catch {
             return undefined;
         }
         return data;
     }
-    
-    /**
-     * Build a page.
-     * 
-     * @param path standard name for page
-     * @returns newly built page, or undefined
-     */
-    private buildPage(name: string): Page {
-        name = this.convertNameToStandard(name);
-        let data: string;
-        if (this.pages[name]?.raw) {
-            data = this.pages[name]?.raw
-        } else {
-            data = this.loadRaw(name)
-        }
-
-        const result = parser.parse(this, data);
-        const title = result.metadata.displayTitle ?? name
-        const content = `${result.metadata.notitle ? '' : `<h1>${title}</h1>`}${result.html}`;
-    
-        const page: Page = {
-            html: content,
-            raw: data,
-            standardName: name,
-            buildTime: Date.now(),
-            metadata: {
-                includeInNavbar: result.metadata.primary ?? false,
-                sortOrder: result.metadata.sortOrder ?? -1,
-                showTitle: !result.metadata.notitle ?? true,
-                displayTitle: title,
-                dependencies: [],
-                dependents: [],
-                errors: []
-            }
-        };
-        this.pages[name] = page;
-        return page;
-    }
-
-    /**
-     * Convert a page name to a standard name.
-     * A standard name is the key used by the page directory.
-     * 
-     * @param name non-standard name for a page
-     */
-    private convertNameToStandard(name: string): string {
-        name = name.replace(/[^a-z0-9:]/gi, '_').toLowerCase();
-        if (!name.includes(':')) {
-            name = `main:${name}`;
-        }
-        return name;
-    }
-
-    /**
-     * Convert a standard name to a file path.
-     * 
-     * @param name standard name for a page
-     */
-    private convertStandardToFilePath(name: string): string {
-        const [first, second] = name.replace('main:', '').split(':');
-        const [title, subpage] = ((second) ? second : first).split('.')
-        const namespace = (second) ? first : undefined
-
-        return `${namespace ? `${namespace}/` : ''}${title}${subpage ? `.${subpage}` : ''}.wiki`
-    }
 }
 
 export type Page = {
@@ -280,8 +278,8 @@ export type PageMetadata = {
     sortOrder?: number;
     showTitle?: boolean;
     includeInNavbar?: boolean;
-    dependencies: string[];
-    dependents: string[];
+    dependencies: Set<string>;
+    dependents: Set<string>;
     errors: PageError[];
 };