convert outputs to schema-driven-processor based yaml files (#51)

* refine memory cache model as pre-step for generating SDP based yaml * add SDP models * fix comment * write SDP based content files and toc to disk * refine test cases and expected results * using double quotes to make sure some Java key words will be handled as string in docs build when they are uses as "name" or "description" * resolve the {@link} in members summary * sort items in toc.yml * sort every level of toc.yml (#50)
docascode · Sep 17, 2020 · d2cd520 · d2cd520
1 parent efc79d0
commit d2cd520
Show file tree

Hide file tree

Showing 163 changed files with 3,884 additions and 4,469 deletions.
diff --git a/pom.xml b/pom.xml
@@ -17,6 +17,7 @@
     <apache.commons-lang.version>3.8.1</apache.commons-lang.version>
     <apache.commons-collections.version>4.2</apache.commons-collections.version>
     <apache.commons-io.version>2.6</apache.commons-io.version>
+    <apache.commons-text.version>1.9</apache.commons-text.version>
     <remark.version>1.1.0</remark.version>
   </properties>
 
@@ -111,6 +112,11 @@
       <artifactId>commons-io</artifactId>
       <version>${apache.commons-io.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-text</artifactId>
+      <version>${apache.commons-text.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>

diff --git a/src/main/java/com/microsoft/build/YmlFilesBuilder.java b/src/main/java/com/microsoft/build/YmlFilesBuilder.java
@@ -5,6 +5,7 @@
 import com.microsoft.lookup.ClassLookup;
 import com.microsoft.lookup.PackageLookup;
 import com.microsoft.model.*;
+import com.microsoft.model.sdp.file.*;
 import com.microsoft.util.ElementUtil;
 import com.microsoft.util.FileUtil;
 import com.microsoft.util.Utils;
@@ -75,21 +76,27 @@ public boolean build() {
 
         populateUidValues(packageMetadataFiles, classMetadataFiles);
 
-        packageMetadataFiles.forEach(FileUtil::dumpToFile);
-        classMetadataFiles.forEach(FileUtil::dumpToFile);
+        // write content files
+        packageMetadataFiles.forEach(YmlFilesBuilder::writePackageContentToYml);
+        for (MetadataFile type : classMetadataFiles)
+        {
+            writeTypeContentToYml(type);
+        }
+
+        // write toc.yml
         FileUtil.dumpToFile(tocFile);
 
         return true;
     }
 
-    void buildFilesForInnerClasses(Element element, List<TocItem> listToAddItems, List<MetadataFile> container) {
+    void buildFilesForInnerClasses(Element element, Set<TocItem> listToAddItems, List<MetadataFile> container) {
         for (TypeElement classElement : elementUtil.extractSortedElements(element)) {
             String uid = classLookup.extractUid(classElement);
             String name = classLookup.extractTocName(classElement);
+            TocItem tocNode = new TocItem(uid, name);
 
-            listToAddItems.add(new TocItem(uid, name));
-
-            container.add(buildClassYmlFile(classElement));
+            container.add(buildClassYmlFile(classElement, tocNode));
+            listToAddItems.add(tocNode);
             buildFilesForInnerClasses(classElement, listToAddItems, container);
         }
     }
@@ -105,12 +112,12 @@ MetadataFile buildPackageMetadataFile(PackageElement packageElement) {
         return packageMetadataFile;
     }
 
-    void addChildrenReferences(Element element, List<String> packageChildren,
+    void addChildrenReferences(Element element, List<MetadataFileItem> packageChildren,
                                Set<MetadataFileItem> referencesCollector) {
         for (TypeElement classElement : elementUtil.extractSortedElements(element)) {
             referencesCollector.add(buildClassReference(classElement));
 
-            packageChildren.add(classLookup.extractUid(classElement));
+            packageChildren.add(classLookup.extractItem(classElement));
             addChildrenReferences(classElement, packageChildren, referencesCollector);
         }
     }
@@ -132,13 +139,13 @@ <T extends Element> void populateItemFields(MetadataFileItem item, BaseLookup<T>
         item.setContent(lookup.extractContent(element));
     }
 
-    MetadataFile buildClassYmlFile(TypeElement classElement) {
+    MetadataFile buildClassYmlFile(TypeElement classElement, TocItem tocItem) {
         String fileName = classLookup.extractHref(classElement);
         MetadataFile classMetadataFile = new MetadataFile(outputPath, fileName);
         addClassInfo(classElement, classMetadataFile);
-        addConstructorsInfo(classElement, classMetadataFile);
-        addMethodsInfo(classElement, classMetadataFile);
-        addFieldsInfo(classElement, classMetadataFile);
+        addConstructorsInfo(classElement, classMetadataFile, tocItem);
+        addMethodsInfo(classElement, classMetadataFile, tocItem);
+        addFieldsInfo(classElement, classMetadataFile, tocItem);
         addReferencesInfo(classElement, classMetadataFile);
         applyPostProcessing(classMetadataFile);
         return classMetadataFile;
@@ -158,28 +165,28 @@ void addClassInfo(TypeElement classElement, MetadataFile classMetadataFile) {
         classMetadataFile.getItems().add(classItem);
     }
 
-    void addChildren(TypeElement classElement, List<String> children) {
-        collect(classElement, children, ElementFilter::constructorsIn, classItemsLookup::extractUid);
-        collect(classElement, children, ElementFilter::methodsIn, classItemsLookup::extractUid);
-        collect(classElement, children, ElementFilter::fieldsIn, classItemsLookup::extractUid);
-        collect(classElement, children, ElementFilter::typesIn, String::valueOf);
+    void addChildren(TypeElement classElement, List<MetadataFileItem> children) {
+        collect(classElement, children, ElementFilter::constructorsIn, classItemsLookup::extractItem);
+        collect(classElement, children, ElementFilter::methodsIn, classItemsLookup::extractItem);
+        collect(classElement, children, ElementFilter::fieldsIn, classItemsLookup::extractItem);
+        collect(classElement, children, ElementFilter::typesIn, classItemsLookup::extractItem);
     }
 
     List<? extends Element> filterPrivateElements(List<? extends Element> elements) {
         return elements.stream()
                 .filter(element -> !Utils.isPrivateOrPackagePrivate(element)).collect(Collectors.toList());
     }
 
-    void collect(TypeElement classElement, List<String> children,
+    void collect(TypeElement classElement, List<MetadataFileItem> children,
                  Function<Iterable<? extends Element>, List<? extends Element>> selectFunc,
-                 Function<? super Element, String> mapFunc) {
+                 Function<? super Element, MetadataFileItem> mapFunc) {
 
         List<? extends Element> elements = selectFunc.apply(classElement.getEnclosedElements());
         children.addAll(filterPrivateElements(elements).stream()
                 .map(mapFunc).collect(Collectors.toList()));
     }
 
-    void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFile) {
+    void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFile, TocItem tocItem) {
         for (ExecutableElement constructorElement : ElementFilter.constructorsIn(classElement.getEnclosedElements())) {
             MetadataFileItem constructorItem = buildMetadataFileItem(constructorElement);
             constructorItem.setOverload(classItemsLookup.extractOverload(constructorElement));
@@ -189,10 +196,15 @@ void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFil
 
             addParameterReferences(constructorItem, classMetadataFile);
             addOverloadReferences(constructorItem, classMetadataFile);
+
+            if(!classElement.getKind().equals(ElementKind.ENUM)){
+                tocItem.getItems().add(
+                        new TocItem(constructorItem.getOverload(), constructorItem.getShortName(), constructorItem.getType().toLowerCase()));
+            }
         }
     }
 
-    void addMethodsInfo(TypeElement classElement, MetadataFile classMetadataFile) {
+    void addMethodsInfo(TypeElement classElement, MetadataFile classMetadataFile, TocItem tocItem) {
         ElementFilter.methodsIn(classElement.getEnclosedElements()).stream()
                 .filter(methodElement -> !Utils.isPrivateOrPackagePrivate(methodElement))
                 .forEach(methodElement -> {
@@ -209,10 +221,15 @@ void addMethodsInfo(TypeElement classElement, MetadataFile classMetadataFile) {
                     addParameterReferences(methodItem, classMetadataFile);
                     addReturnReferences(methodItem, classMetadataFile);
                     addOverloadReferences(methodItem, classMetadataFile);
+
+                    if(!classElement.getKind().equals(ElementKind.ENUM)){
+                        tocItem.getItems().add(
+                                new TocItem(methodItem.getOverload(), methodItem.getShortName(), methodItem.getType().toLowerCase()));
+                    }
                 });
     }
 
-    void addFieldsInfo(TypeElement classElement, MetadataFile classMetadataFile) {
+    void addFieldsInfo(TypeElement classElement, MetadataFile classMetadataFile,TocItem tocItem) {
         ElementFilter.fieldsIn(classElement.getEnclosedElements()).stream()
                 .filter(fieldElement -> !Utils.isPrivateOrPackagePrivate(fieldElement))
                 .forEach(fieldElement -> {
@@ -221,6 +238,11 @@ void addFieldsInfo(TypeElement classElement, MetadataFile classMetadataFile) {
                     fieldItem.setReturn(classItemsLookup.extractReturn(fieldElement));
                     classMetadataFile.getItems().add(fieldItem);
                     addReturnReferences(fieldItem, classMetadataFile);
+
+                    if(classElement.getKind() != ElementKind.ENUM) {
+                        tocItem.getItems().add(
+                                new TocItem(fieldItem.getUid(), fieldItem.getName(), fieldItem.getType().toLowerCase()));
+                    }
                 });
     }
 
@@ -307,6 +329,33 @@ void applyPostProcessing(MetadataFile classMetadataFile) {
         expandComplexGenericsInReferences(classMetadataFile);
     }
 
+    private static void writePackageContentToYml(MetadataFile pack) {
+        for (MetadataFileItem item : pack.getItems()) {
+            PackageModel model = new PackageModel(item, pack.getFileNameWithPath());;
+            FileUtil.dumpToFile(model);
+        }
+    }
+
+    private void writeTypeContentToYml(MetadataFile typeItem) {
+        MetadataFileItem item = typeItem.getItems().iterator().next();
+        String type = item.getType();
+        switch (type.toLowerCase()) {
+            case "class":
+            case "interface":
+                TypeModel typeModel = new TypeModel(item, typeItem.getFileNameWithPath(), getOutputPath());
+                FileUtil.dumpToFile(typeModel);
+                break;
+            case "enum":
+                EnumModel enumModel = new EnumModel(item, typeItem.getFileNameWithPath());
+                FileUtil.dumpToFile(enumModel);
+                break;
+        }
+    }
+
+    public String getOutputPath() {
+        return outputPath;
+    }
+
     /**
      * Replace one record in 'references' with several records in this way:
      * <pre>
@@ -346,6 +395,12 @@ void populateUidValues(List<MetadataFile> packageMetadataFiles, List<MetadataFil
                         populateUidValues(item.getSummary(), lookupContext)
                 ));
 
+                for(MetadataFileItem child :item.getChildren()) {
+                    child.setSummary(YamlUtil.convertHtmlToMarkdown(
+                            populateUidValues(child.getSummary(), lookupContext)
+                    ));
+                }
+
                 Optional.ofNullable(item.getSyntax()).ifPresent(syntax -> {
                             Optional.ofNullable(syntax.getParameters()).ifPresent(
                                     methodParams -> methodParams.forEach(

diff --git a/src/main/java/com/microsoft/lookup/BaseLookup.java b/src/main/java/com/microsoft/lookup/BaseLookup.java
@@ -45,6 +45,10 @@ protected BaseLookup(DocletEnvironment environment) {
         this.environment = environment;
     }
 
+    public ExtendedMetadataFileItem extractItem(T key) {
+        return resolve(key);
+    }
+
     protected ExtendedMetadataFileItem resolve(T key) {
         ExtendedMetadataFileItem value = map.get(key);
         if (value == null) {

diff --git a/src/main/java/com/microsoft/lookup/ClassItemsLookup.java b/src/main/java/com/microsoft/lookup/ClassItemsLookup.java
@@ -4,6 +4,7 @@
 
 import com.microsoft.model.ExceptionItem;
 import com.microsoft.model.MethodParameter;
+import com.microsoft.model.Field;
 import com.microsoft.model.Return;
 
 import com.microsoft.util.CommentHelper;
@@ -81,7 +82,7 @@ protected ExtendedMetadataFileItem buildMetadataFileItem(Element element) {
         if (element instanceof VariableElement) {
             String type = makeTypeShort(String.valueOf(element.asType()));
             result.setFieldContent(String.format("%s %s %s", modifiers, type, elementQName));
-            result.setReturn(extractReturn((VariableElement) element));
+            result.setField(extractField((VariableElement) element));
         }
         return result;
     }
@@ -154,6 +155,11 @@ String extractOverriddenUid(ExecutableElement ovr) {
         return "";
     }
 
+    public Field extractField(VariableElement fieldElement) {
+        var constantValue = (null == fieldElement.getConstantValue()) ? null : fieldElement.getConstantValue().toString();
+        return new Field(String.valueOf(fieldElement.asType()), constantValue);
+    }
+
     /**
      * If the item being inherited from is declared from external compiled package,
      * or is declared in the packages like java.lang.Object,

diff --git a/src/main/java/com/microsoft/lookup/model/ExtendedMetadataFileItem.java b/src/main/java/com/microsoft/lookup/model/ExtendedMetadataFileItem.java
@@ -116,4 +116,20 @@ public void addReferences(Set<MetadataFileItem> references) {
     public Set<MetadataFileItem> getReferences() {
         return references;
     }
+
+    public String getSyntaxContent() {
+        String content = "";
+        switch (this.getType().toLowerCase()) {
+            case "constructor":
+                content = getConstructorContent();
+                break;
+            case "field":
+                content = getFieldContent();
+                break;
+            case "method":
+                content = getMethodContent();
+                break;
+        }
+        return content;
+    }
 }
diff --git a/src/main/java/com/microsoft/model/ExceptionItem.java b/src/main/java/com/microsoft/model/ExceptionItem.java
@@ -1,13 +1,22 @@
 package com.microsoft.model;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import com.microsoft.util.XrefHelper;
+
 public class ExceptionItem {
 
+    @JsonIgnore
     private final String type;
+    @JsonProperty("type")
+    private String exceptionXrefString;
     private final String description;
 
     public ExceptionItem(String type, String description) {
         this.type = type;
         this.description = description;
+        this.exceptionXrefString = XrefHelper.generateXrefString(type, XrefHelper.XrefOption.SHORTNAME);
     }
 
     public String getType() {
@@ -17,4 +26,8 @@ public String getType() {
     public String getDescription() {
         return description;
     }
+
+    public String getExceptionXrefString() {
+        return exceptionXrefString;
+    }
 }
diff --git a/src/main/java/com/microsoft/model/Field.java b/src/main/java/com/microsoft/model/Field.java
@@ -0,0 +1,34 @@
+package com.microsoft.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.microsoft.util.XrefHelper;
+
+public class Field {
+
+    @JsonIgnore
+    private final String fieldType;
+    @JsonProperty("description")
+    private String fieldDescription;
+    @JsonProperty("type")
+    private String fieldXrefString;
+    private String value;
+
+    public String getFieldDescription() {
+        return fieldDescription;
+    }
+
+    public String getFieldXrefString() {
+        return fieldXrefString;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    public Field(String fieldType, String value) {
+        this.fieldType = fieldType;
+        this.value = value ;
+        this.fieldXrefString = XrefHelper.generateXrefString(fieldType, XrefHelper.XrefOption.SHORTNAME);
+    }
+}
diff --git a/src/main/java/com/microsoft/model/MetadataFile.java b/src/main/java/com/microsoft/model/MetadataFile.java
@@ -32,7 +32,7 @@ public Set<MetadataFileItem> getReferences() {
     public String getFileContent() {
         Set<MetadataFileItem> sortedSet = new TreeSet<>(this.items);
         this.items = sortedSet;
-        return METADATA_FILE_HEADER + YamlUtil.objectToYamlString(this);
+        return METADATA_FILE_HEADER + YamlUtil.objectToYamlString(this, this.fileName);
     }
 
     @JsonIgnore