From 0a74291cae20b3b729b485b487a36eec40680c8b Mon Sep 17 00:00:00 2001 From: erdos Date: Fri, 12 Aug 2022 15:16:29 -0400 Subject: [PATCH 1/8] fix: numberings in loops are reset --- src/stencil/model.clj | 9 +++++++++ src/stencil/postprocess/list_ref.clj | 18 ++++++++++++++++++ src/stencil/tree_postprocess.clj | 2 ++ 3 files changed, 29 insertions(+) diff --git a/src/stencil/model.clj b/src/stencil/model.clj index 66e0a6bd..8ea802c1 100644 --- a/src/stencil/model.clj +++ b/src/stencil/model.clj @@ -6,6 +6,7 @@ (:require [clojure.data.xml :as xml] [clojure.java.io :as io :refer [file]] [stencil.eval :as eval] + [stencil.ooxml :as ooxml] [stencil.merger :as merger] [stencil.model.numbering :as numbering] [stencil.types :refer [->FragmentInvoke ->ReplaceImage]] @@ -52,10 +53,18 @@ {:source-file cts ::path (.getName cts)}) +(defn- add-unique-index + "Annotates some elements with an unique id. + These elements need special care when rendering duplicates them. + For example, numberings need to be reset to start from 1 again." + [elem] + (when (= (:open+close elem) ooxml/attr-numId) + (update elem :attrs assoc ::unique (gensym "uniq")))) (defn ->exec [xml-streamable] (with-open [stream (io/input-stream xml-streamable)] (-> (merger/parse-to-tokens-seq stream) + (->> (map (some-fn add-unique-index identity))) (cleanup/process) (select-keys [:variables :dynamic? :executable :fragments])))) diff --git a/src/stencil/postprocess/list_ref.clj b/src/stencil/postprocess/list_ref.clj index cc2ff9b3..0066fc1c 100644 --- a/src/stencil/postprocess/list_ref.clj +++ b/src/stencil/postprocess/list_ref.clj @@ -307,3 +307,21 @@ (let [xml-tree (enrich-dirty-refs-meta xml-tree) bookmark-meta (get-bookmark-meta xml-tree)] (rerender-refs xml-tree bookmark-meta))) + +;; finds nodes with unique id and if these are not unique anymore, replace the numbering id with a duplicate numbering definition +(defn fix-numbering-reset [xml-tree] + (let [found (atom #{}) + last-val (atom nil) + edit-attrs (fn [attrs] + (let [old-id (ooxml/val attrs) + unq-id (:stencil.model/unique attrs) + new-id old-id] ;; TODO + ;; ha az unique id benne van a found set-ben + ;; akkor uj value-t szamolunk es beallitjuk a last-val-t. + ;; egyebkent ha a last-val nem ugyanaz mint a mostani, akkor beallitjuk a last val-t (?) + (-> attrs + (dissoc :stencil.model/unique) + (assoc ooxml/val new-id))))] + ;; find nodes with unique id in attrs + (dfs-walk-xml-node xml-tree (fn [n] (and (map? n) (:stencil.model/unique (:attrs n)))) + (fn [n] (zip/edit n update :attrs edit-attrs))))) diff --git a/src/stencil/tree_postprocess.clj b/src/stencil/tree_postprocess.clj index 83bb1e97..2cb5f7b8 100644 --- a/src/stencil/tree_postprocess.clj +++ b/src/stencil/tree_postprocess.clj @@ -26,6 +26,8 @@ #'fix-list-dirty-refs + #'fix-numbering-reset + #'replace-images ;; call this first. includes fragments and evaluates them too. From b5c228d44dfee1c0bcd40b2db4f9eab71115df54 Mon Sep 17 00:00:00 2001 From: erdos Date: Sun, 25 Dec 2022 13:09:55 +0100 Subject: [PATCH 2/8] wip --- src/stencil/cleanup.clj | 115 +++++++++++++---------- test-resources/test-numbering-loops.docx | Bin 0 -> 5705 bytes 2 files changed, 67 insertions(+), 48 deletions(-) create mode 100644 test-resources/test-numbering-loops.docx diff --git a/src/stencil/cleanup.clj b/src/stencil/cleanup.clj index 43d8626e..b622cef6 100644 --- a/src/stencil/cleanup.clj +++ b/src/stencil/cleanup.clj @@ -8,6 +8,7 @@ valid XML String -> tokens -> Annotated Control AST -> Normalized Control AST -> Evaled AST -> Hiccup or valid XML String " (:require [stencil.util :refer [mod-stack-top-conj mod-stack-top-last parsing-exception stacks-difference-key]] + [stencil.ooxml :as ooxml] [stencil.types :refer [open-tag close-tag]])) (set! *warn-on-reflection* true) @@ -77,24 +78,24 @@ (assert (sequential? control-ast)) (let [stack (volatile! ())] (mapv (partial nested-tokens-fmap-postwalk - (fn before-cmd-block [block] - (assoc block ::before @stack)) - - (fn after-cmd-block [block] - (let [stack-before (::before block) - [a b] (stacks-difference-key :open stack-before @stack)] - (assoc block ::before a ::after b))) - - (fn child [item] - (cond - (:open item) - (vswap! stack conj item) - - (:close item) - (if (= (:close item) (:open (first @stack))) - (vswap! stack next) - (throw (ex-info "Unexpected stack state" {:stack @stack, :item item})))) - item)) + (fn before-cmd-block [block] + (assoc block ::before @stack)) + + (fn after-cmd-block [block] + (let [stack-before (::before block) + [a b] (stacks-difference-key :open stack-before @stack)] + (assoc block ::before a ::after b))) + + (fn child [item] + (cond + (:open item) + (vswap! stack conj item) + + (:close item) + (if (= (:close item) (:open (first @stack))) + (vswap! stack next) + (throw (ex-info "Unexpected stack state" {:stack @stack, :item item})))) + item)) control-ast))) (defn stack-revert-close @@ -166,42 +167,43 @@ ;; amikor van benne blocks ;; mapping: {Sym -> Str} (letfn [(resolve-sym [mapping s] - (assert (map? mapping)) - (assert (symbol? s)) - (mapping s (name s))) + (assert (map? mapping)) + (assert (symbol? s)) + (mapping s (name s))) (expr [mapping e] - (cond (symbol? e) [(resolve-sym mapping e)] - (not (sequential? e)) nil - (= :fncall (first e)) (mapcat (partial expr mapping) (nnext e)) - (= :get (first e)) (let [[ss rest] (split-with string? (nnext e))] - (cons - (reduce (fn [root item] (str root "." item)) - (resolve-sym mapping (second e)) - ss) - (mapcat (partial expr mapping) rest))) - :else (mapcat (partial expr mapping) (next e)))) + (cond (symbol? e) [(resolve-sym mapping e)] + (not (sequential? e)) nil + (= :fncall (first e)) (mapcat (partial expr mapping) (nnext e)) + (= :get (first e)) (let [[ss rest] (split-with string? (nnext e))] + (cons + (reduce (fn [root item] (str root "." item)) + (resolve-sym mapping (second e)) + ss) + (mapcat (partial expr mapping) rest))) + :else (mapcat (partial expr mapping) (next e)))) (maybe-variable [mapping e] - (cond (symbol? e) - (resolve-sym mapping e) - (and (sequential? e) (= :get (first e)) (symbol? (second e)) (every? string? (nnext e))) - (reduce (fn [a b] (str a "." b)) (resolve-sym mapping (second e)) (nnext e)))) + (cond (symbol? e) + (resolve-sym mapping e) + (and (sequential? e) (= :get (first e)) (symbol? (second e)) (every? string? (nnext e))) + (reduce (fn [a b] (str a "." b)) (resolve-sym mapping (second e)) (nnext e)))) (collect [m xs] (mapcat (partial collect-1 m) xs)) (collect-1 [mapping x] - (case (:cmd x) - :cmd/echo (expr mapping (:expression x)) - - :if (concat (expr mapping (:condition x)) - (collect mapping (apply concat (::blocks x)))) - - :for (let [variable (maybe-variable mapping (:expression x)) - exprs (expr mapping (:expression x)) - mapping (if variable - (assoc mapping (:variable x) (str variable "[]")) - mapping)] - (concat exprs (collect mapping (apply concat (::blocks x))))) - []))] + (case (:cmd x) + :cmd/echo (expr mapping (:expression x)) + + :if (concat (expr mapping (:condition x)) + (collect mapping (apply concat (::blocks x)))) + + :for (let [variable (maybe-variable mapping (:expression x)) + exprs (expr mapping (:expression x)) + mapping (if variable + (assoc mapping (:variable x) (str variable "[]")) + mapping)] + (concat exprs (collect mapping (apply concat (::blocks x))))) + []))] (distinct (collect {} control-ast)))) +;; TODO: does it work? (defn- find-fragments [control-ast] ;; returns a set of fragment names use in this document (set (for [item (tree-seq map? (comp flatten ::blocks) {::blocks [control-ast]}) @@ -209,11 +211,28 @@ :when (= :cmd/include (:cmd item))] (:name item)))) +;; Calculate for each numbering how many for loops deep it is. +(defn- numid->depth [ast] + (let [cache (volatile! {})] + ;; visit all nodes and set up a cache. + (doseq [e ast] + ((fn f [d node] + (assert (map? node)) + (when (= ooxml/attr-numId (:open+close node)) + (vswap! cache update (ooxml/val (:attrs node)) (fnil conj #{}) d)) + (let [d (if (= :for (:cmd node)) (cons (gensym "x") d) d)] + (doseq [block (::blocks node) + b (::children block)] + (f d b)))) () e)) + (into {} (for [[k v] @cache] + [k (count (take-while true? (apply map = (map reverse v))))])))) + (defn process [raw-token-seq] (let [ast (tokens->ast raw-token-seq) executable (mapv control-ast-normalize (annotate-environments ast))] {:variables (find-variables ast) :fragments (find-fragments ast) + :num-depth (numid->depth ast) :dynamic? (boolean (some :cmd executable)) :executable executable})) diff --git a/test-resources/test-numbering-loops.docx b/test-resources/test-numbering-loops.docx new file mode 100644 index 0000000000000000000000000000000000000000..01d8b5e359c9d273cf2cbd9d9e4dad0262598533 GIT binary patch literal 5705 zcma)A1ymGF*QP;28YDy-q(wwPkw#$!Ryw4+yFp+9>0T*mSR^ErP-5v`SU?)2Yehn& z(|_0h{XWs}|2+SlbLPyRoo9FExpVK_d4K>+EDAI{JUp~8qcjb)D@K4E8@ia=x$^U& z`m)4XMNBEe(48|1neB!tO_GGntr$Egc`-4G#rp=C2NGuTWsWsj^`ozJlLw{S2`cI(tSWrw8lKYQ08x;GAwpgb-dCvo__*NzFXu$^-}< zl3jF9v|;XaE~jB-p(b)a%dOoD%PU3?OsT64SgqGvSaMx7n&M-5rU;PL`k^4C<=5UQ z|L)j*NhMt)tjZBFj}8Rj;-!{#;?E-YEQ^7L2K=wAkRoT8Ihq1p9GzVGO&wj#`Mm7y zVpL(u?E)l?dz1+~5lPh#&De^uia5BKYVbM+CTB)T9wcuGZ_O{WD(U9nIy`u=Je5*4 zWuV!5J({wn?-^MqCt9ZspL$w;%Gkq3nILy{XWcP;7dNyhi%yL$A>*1LOEp^oi2DK1{8>!s_F z|5Cc9)Fd!+a}v)OrwEh53-eP7UFt+{`?5`Jq>%E6fI|De&#MA6F{a%u-zz&KHDix7{?m*OWZi_UTybq56elA-)CWx>K}HOEArv}OsfID)!d7VrKG zS5}EZFzZNF0#0!G`=Cs#9|j2?a*roH>_6PfZh0>83D0jcq%=o`crp8jwjUuZLzb*lt64qNMVfv zj_!7Cb@EX16KTquZ-!RZEAEe)dHJZkGbo3q<*^o`FGEAG<=s{%c$jI_q_sIbEw4c8 z#O7DlMBg~;Gjt{rweBpY%kBSgM}m6q!jF~+|M2*wV|6ja6>Ww zPx64_w+WEQ9lelCg({UYGb_!8FKBMycDS9ktVsz*UP943al=0%TpR?CvieyK_1y;H z2pP+mKpusl+-J2LPg#r%9l8j#Df9ippaouR)tLl5hpCJ?Y%TzJu+1}AymlCS*66sc z!n=JUPt}{|!G4K_uG{nCImh43W)uVGHEAlpZ+yGus5g<>R-?fY^Rh|yo8$z8j2pN~ zip;4Z34wX;dv4a_<@yPC-#MNi^k$#HM2uC|p6J#0p=E-l+ZUizqIx{m$ha4Voig5x z(Jq&}?U8gSemeYu3dfPVPyH)DsI2jmpR3Gl>FTL0i%`6nH#C-8B2Dz@v!PIu>Sxs4 zYr!ktBz7X>*o}64pP9UE_2Z0TeHwd!69Typl2!8C+sQEGg82W@DcZzW<=5pT{IF#Z0H_y2-QIawOAZwFt5cs(b%)ShE3%519iJefd za$-e24affS!G$Y4-974&F10F$A}sX=X3)ahPp~_by#1Vr#maVT&!R`^F$qwqP-NAfe zY5Ia7szv&$;-i7kSnn5g>Fm{M#3jW$&Ueu{G-qs7A22r4*&1C)*WzuBvD~4G06mtK z>5opDvXQItj9a@ILdr*fw~w;ORJFuO3mvqmjBhI$tMv4?2h~iw41FJ%HGU{p03k?VnJCXl!)G(s(|}FE$9@h zCAQ{rx!IN@tMUd!?gxGK%*<17vpi@{tPb{~yl0Fqr3WtQ_N8ttZmeoj4Y!a9n_p`$ zvz_O4ZYTWQ0LZ&ni^yWFUf?co0+*$P4eIr@b8qT(hrhDpf!S?_XZv|QJ~!ihm%q%# z;a)19*K;c%E&%&Nn!v`jK*6}Dr2~iEqx$Jv&%Nf)3VSLrerOS!pVh@qwElKs3yz1k z2%Cp2{d&iAH>up^u{U5|Z^Sa4DfAv(Nb@BA^-br#B(aIJ6HvKh&>$;^ekcndYm@!Q z)aC$>`~zbui078xb&j=^YSCf6-EAei#-JBCRFUG3B8kBc{3=Ati^YqXuO=PpY_su2 z?GimqC@j6)L@OVsFyV4<(~-^8Yp0aB>!XeMx9Iq!3!@=Vb&<{jqqbC76N@HK{PBtuj^V9eHKD>apx2W1O!uZnNu z9*1;PD<%0BpGv_Fn-WYcY&ht>n<2EHA*wgq0Z2*NDuTeLwuz_?Z?rt){3x zVxplj3jD2DApTd_ySjPXnY*IWVW+yj%8USo|9nOGxq&K)NfEOg58g15`v|^7i@O6I zLHD6uSnc_#KF1xvYjB(m8Na<>hOfULychcMIW@BnhG|{mHPJ`3iFmAPG%2u$!?_&m#h`A@`&Vi zkvGo)+L`snaR$M*FeMEcrJazDqB-LEf_(xp&9eU8&T?ZZz~WoObi+a&;nIyAc?HY- zv2fikZ|Uu0dcs;Q!StX-0+8Oxt7_lcKwVFBQKcSrQObd$>Sw1HkA(mid{x>Im+d@T zue?OQ;zicO?nj`6<2G{2Yof(WXXh9hIIJ}dsW~n*0`b9NQ^HZ}e(}oULhCfD@Dn4! zgHUJJMK3KJXv6ma?yO>s$hI47Htdjb984z-G53aH(H?3OICv-{=Qa7X_;M_f2FAN- zdm$f4aV|i{eOp0GFR4E&qBg2eR)GM|@}862mq8`UnB#lX1|h%)c1h=UXK7@8SQZp= z7yIh!pFFpWMNix5Q|*2GFlt(?1!gE*+3Y5YzDZNeA-u*Tmu<05_nZkv2kYT==oPbX zsNQG4=YL2x&bM%Me6D2v8v4!$YiLAM>dC|J!{A-T6VZhN49A`!atHyK5O=S)TP=)q zn!Mg(eW<@C(QPiNVDAV!Z$6Hi$dhQ-XvK5By5Jciy8r|@)<5+&@Mk0fJ#EfPNM6$O z$v*tzkAQ4{bk8KvFq+q|H65i5^>5m7{Q@kxJ@$y@W-WRE;5=(IEFmB)Zz8*DHpx5t zJF(e|0BP@w*VCRVO)>}RJ%r;oYi#zLt=+@1wxli_4R)e;X^X->8t!wEWNjYvqi#Ih zVkx60JegrR>{%HSUb$S0fyZQMx32Wbo^FsTI@~4XOo;_uOB{a`efq9D7}^e;Y>ZYlGW~- z5S=MRoWVxCV&zqhYrC*5S>+sM^SW2V!j?f>Un21`?|Wv?r$*LsTCazLYyFbxtil0#1G!+mX2Kr}qn?ML0VT zH)hWj=|1KVWe(w7evY(V^7odZZcLz?6RF+P+-+)1AefsZ+WGw9=ZAL$bH(Iy0#DvH zvd^V>Zh-DPw?C+9E`Qp{@3;i=Tzl#15MY?*WjL2cFjq%D$64K6f&6OcGxiS0{l`|G*!Me&-C%$TbIbX*#Eybaf&XZZFf z)Iu63YTLRvHEE)u>4KU^ z=;d2uIN#r_sFu#Zc&BmWZgg%Fz$InXV|a!(YxE+2P|)^P5QfitDI0jBxlcN4RX0r>js{Sg0q(iUD2%1M*th1_ zgvp%~??oub(ub}30vMVgmDu15JqA~fQOc?DhnHh{Q)5pX19={EZv`CPL* zJ8t~`%d$$c%=#%o?Nw{3C`Xiw#5oeWO_47b2-e?R^zX{t)!fYunWcVS(&q8v4($Sj zp+{M5Lk6(xLlP_=wRwOy1-M3LVpNO~AlYtr6~jXLg|6uKHI zuebzmY}kLS-uSwId0Z0NPvYZ5QK>;-;f`Q9bWKXoQLl?9V!H2ID!(>ql{pa3;G#ep zTp5&zn`~K+NK?H76fr{bQNH9LPeV4*#pJZNS=X0*A8?Y81#zp?3^kF!e`9xaH<31xzk*d%!6-ol6XZYQKeRyGkEzK!V+7(94H5Y zOr}MRkj`1gMmY!_g97cBaCcR_LJ4=j+Z7q_PvBKa1r-awj0*Wz|DkmK3BM{2pmdjC zR)@@v|Azmg!u*N8`glf32)~R68MJ>B6#o3BtM&Nr+jxp(>pz~}f5NZUG^nEem&siF z2mY&s{}bGC Date: Sun, 25 Dec 2022 17:46:58 +0100 Subject: [PATCH 3/8] wip --- src/stencil/cleanup.clj | 31 +++++++++++++++++++++++++++++++ src/stencil/eval.clj | 30 ++++++++++++++++++------------ src/stencil/model.clj | 3 ++- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/stencil/cleanup.clj b/src/stencil/cleanup.clj index b622cef6..d36c00dc 100644 --- a/src/stencil/cleanup.clj +++ b/src/stencil/cleanup.clj @@ -227,8 +227,39 @@ (into {} (for [[k v] @cache] [k (count (take-while true? (apply map = (map reverse v))))])))) +;; add ::depth to ooxml/numId elements +(defn- ast-numbering-depths [ast] + (let [numid->paths (volatile! {}) + numid->depth (memoize (fn [id] + (->> (get @numid->paths id) + (map reverse) + (apply map =) + (take-while true?) + (count))))] + (letfn [(visit-all [path xs] (doseq [x xs] (visit path x))) + (visit [path x] + (if (= ooxml/attr-numId (:open+close x)) + (vswap! numid->paths update (-> x :attrs ooxml/val) + (fnil conj #{}) path) + (when-let [blocks (::blocks x)] + (let [path (if (= :for (:cmd x)) + (cons (gensym) path) path)] + (doseq [block blocks] + (visit-all path (::children block)))))))] + (visit-all () ast)) + (mapv + (partial nested-tokens-fmap-postwalk + identity identity + (fn [e] + (if (= ooxml/attr-numId (:open+close e)) + (assoc e ::depth (numid->depth (-> e :attrs ooxml/val))) + e))) + ast))) + + (defn process [raw-token-seq] (let [ast (tokens->ast raw-token-seq) + ast (ast-numbering-depths ast) executable (mapv control-ast-normalize (annotate-environments ast))] {:variables (find-variables ast) :fragments (find-fragments ast) diff --git a/src/stencil/eval.clj b/src/stencil/eval.clj index 7e88c250..be506e59 100644 --- a/src/stencil/eval.clj +++ b/src/stencil/eval.clj @@ -5,37 +5,38 @@ [stencil.types :refer [control?]] [stencil.tokenizer :as tokenizer] [stencil.util :refer [eval-exception]] - [stencil.tree-postprocess :as tree-postprocess])) + [stencil.tree-postprocess :as tree-postprocess] + [stencil.ooxml :as ooxml])) (set! *warn-on-reflection* true) -(defmulti eval-step (fn [function data item] (:cmd item))) +(defmulti eval-step (fn [function data trace item] (or (:cmd item) (:open+close item)))) -(defmethod eval-step :default [_ _ item] [item]) +(defmethod eval-step :default [_ _ _ item] [item]) -(defn normal-control-ast->evaled-seq [data function items] +(defn normal-control-ast->evaled-seq [data function trace items] (assert (map? data)) (assert (ifn? function)) (assert (or (nil? items) (sequential? items))) - (eduction (mapcat (partial eval-step function data)) items)) + (eduction (mapcat (partial eval-step function data trace)) items)) (defn- eval-rpn* [data function expr raw-expr] (try (eval-rpn data function expr) (catch Exception e (throw (eval-exception (str "Error evaluating expression: " raw-expr) e))))) -(defmethod eval-step :if [function data item] +(defmethod eval-step :if [function data trace item] (let [condition (eval-rpn* data function (:condition item) (:raw item))] (log/trace "Condition {} evaluated to {}" (:condition item) condition) (->> (if condition (:branch/then item) (:branch/else item)) - (normal-control-ast->evaled-seq data function)))) + (normal-control-ast->evaled-seq data function trace)))) -(defmethod eval-step :cmd/echo [function data item] +(defmethod eval-step :cmd/echo [function data _ item] (let [value (eval-rpn* data function (:expression item) (:raw item))] (log/trace "Echoing {} as {}" (:expression item) value) [{:text (if (control? value) value (str value))}])) -(defmethod eval-step :for [function data item] +(defmethod eval-step :for [function data trace item] (let [items (eval-rpn* data function (:expression item) (:raw item))] (log/trace "Loop on {} will repeat {} times" (:expression item) (count items)) (if (not-empty items) @@ -45,13 +46,18 @@ datas (if (or (instance? java.util.Map items) (map? items)) (map datamapper (keys items) (vals items)) (map-indexed datamapper items)) - bodies (cons (:branch/body-run-once item) (repeat (:branch/body-run-next item)))] - (mapcat (fn [data body] (normal-control-ast->evaled-seq data function body)) datas bodies)) + bodies (cons (:branch/body-run-once item) (repeat (:branch/body-run-next item))) + traces (for [i (range)] (cons i trace))] + (mapcat (fn [data body trace] (normal-control-ast->evaled-seq data function trace body)) datas bodies traces)) (:branch/body-run-none item)))) +(defmethod eval-step ooxml/attr-numId [_ _ trace item] + (println :!!! item) + [item]) + (defn eval-executable [part data functions] (->> (:executable part) (#(doto % assert)) - (normal-control-ast->evaled-seq data functions) + (normal-control-ast->evaled-seq data functions ()) (tokenizer/tokens-seq->document) (tree-postprocess/postprocess))) diff --git a/src/stencil/model.clj b/src/stencil/model.clj index 8ea802c1..e27c5e7e 100644 --- a/src/stencil/model.clj +++ b/src/stencil/model.clj @@ -53,6 +53,7 @@ {:source-file cts ::path (.getName cts)}) +#_ (defn- add-unique-index "Annotates some elements with an unique id. These elements need special care when rendering duplicates them. @@ -64,7 +65,7 @@ (defn ->exec [xml-streamable] (with-open [stream (io/input-stream xml-streamable)] (-> (merger/parse-to-tokens-seq stream) - (->> (map (some-fn add-unique-index identity))) + ;(->> (map (some-fn add-unique-index identity))) (cleanup/process) (select-keys [:variables :dynamic? :executable :fragments])))) From 2fc2473a262a16cd03f2ccb67c57f5df444382c6 Mon Sep 17 00:00:00 2001 From: erdos Date: Sun, 25 Dec 2022 19:44:15 +0100 Subject: [PATCH 4/8] wip --- src/stencil/cleanup.clj | 2 +- src/stencil/eval.clj | 3 +-- src/stencil/model.clj | 5 +++-- src/stencil/model/numbering.clj | 20 +++++++++++++---- src/stencil/postprocess/list_ref.clj | 18 ---------------- src/stencil/postprocess/numberings.clj | 30 ++++++++++++++++++++++++++ src/stencil/tree_postprocess.clj | 3 ++- test/stencil/model/numbering_test.clj | 8 ++++--- 8 files changed, 58 insertions(+), 31 deletions(-) create mode 100644 src/stencil/postprocess/numberings.clj diff --git a/src/stencil/cleanup.clj b/src/stencil/cleanup.clj index d36c00dc..d3408a2d 100644 --- a/src/stencil/cleanup.clj +++ b/src/stencil/cleanup.clj @@ -252,7 +252,7 @@ identity identity (fn [e] (if (= ooxml/attr-numId (:open+close e)) - (assoc e ::depth (numid->depth (-> e :attrs ooxml/val))) + (assoc-in e [:attrs ::depth] (numid->depth (-> e :attrs ooxml/val))) e))) ast))) diff --git a/src/stencil/eval.clj b/src/stencil/eval.clj index be506e59..372c014a 100644 --- a/src/stencil/eval.clj +++ b/src/stencil/eval.clj @@ -52,8 +52,7 @@ (:branch/body-run-none item)))) (defmethod eval-step ooxml/attr-numId [_ _ trace item] - (println :!!! item) - [item]) + [(assoc-in item [:attrs ::trace] trace)]) (defn eval-executable [part data functions] (->> (:executable part) diff --git a/src/stencil/model.clj b/src/stencil/model.clj index e27c5e7e..7db40d57 100644 --- a/src/stencil/model.clj +++ b/src/stencil/model.clj @@ -144,7 +144,7 @@ (assert (:main template-model) "Should be a result of load-template-model call!") (assert (some? fragments)) (binding [*current-styles* (atom (:parsed (:style (:main template-model)))) - numbering/*numbering* (::numbering (:main template-model)) + numbering/*numbering* (atom (::numbering (:main template-model))) *inserted-fragments* (atom #{}) *extra-files* (atom #{}) *all-fragments* (into {} fragments)] @@ -175,6 +175,7 @@ :finally (assoc :result result))))] (-> template-model (update :main evaluate) + (assoc-in [:main ::numbering] @numbering/*numbering*) (update-in [:main :headers+footers] (partial mapv evaluate)) (cond-> (-> template-model :main :style) @@ -240,7 +241,7 @@ elem))) -(defmethod eval/eval-step :cmd/include [function local-data-map {frag-name :name}] +(defmethod eval/eval-step :cmd/include [function local-data-map _ {frag-name :name}] (assert (map? local-data-map)) (assert (string? frag-name)) (expect-fragment-context! diff --git a/src/stencil/model/numbering.clj b/src/stencil/model/numbering.clj index 27c4d99b..b264a0df 100644 --- a/src/stencil/model/numbering.clj +++ b/src/stencil/model/numbering.clj @@ -2,14 +2,14 @@ (:require [clojure.data.xml :as xml] [clojure.java.io :as io] [stencil.ooxml :as ooxml] - [stencil.util :refer [unlazy-tree ->int]] + [stencil.util :refer [unlazy-tree ->int find-first]] [stencil.model.common :refer [unix-path]])) (def ^:private rel-type-numbering "http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering") - +;; swap an atom here! (def ^:dynamic *numbering* nil) @@ -66,7 +66,7 @@ (let [tree (xml/parse r)] (prepare-numbering-xml tree)))) - +;; finds the (defn main-numbering [dir main-document main-document-rels] (when-let [main-numbering-path (some #(when (= rel-type-numbering (:stencil.model/type %)) @@ -81,6 +81,18 @@ (defn style-def-for [id lvl] (assert (string? id)) (assert (integer? lvl)) - (some-> (:parsed *numbering*) + (some-> (:parsed @*numbering*) (get-id-style-xml id lvl) (xml-lvl-parse))) + +;; returns new id for the numbering copied from old-id +(defn copy-numbering! [old-id] + (let [old-elem (find-first (fn [e] (-> e :attrs ooxml/attr-numId (= old-id))) (:content (:parsed @*numbering*))) + new-id #_(name (gensym "xx")) (str (int (* 1000 (Math/random)))) + new-elem (assoc-in old-elem [:attrs ooxml/attr-numId] new-id)] + (assert old-elem) + (swap! *numbering* update :parsed update :content concat [new-elem]) + (swap! *numbering* dissoc :source-file) + (swap! *numbering* (fn [numbering] + (assoc numbering :result {:writer (stencil.model.common/->xml-writer (:parsed numbering))}))) + new-id)) \ No newline at end of file diff --git a/src/stencil/postprocess/list_ref.clj b/src/stencil/postprocess/list_ref.clj index 0066fc1c..cc2ff9b3 100644 --- a/src/stencil/postprocess/list_ref.clj +++ b/src/stencil/postprocess/list_ref.clj @@ -307,21 +307,3 @@ (let [xml-tree (enrich-dirty-refs-meta xml-tree) bookmark-meta (get-bookmark-meta xml-tree)] (rerender-refs xml-tree bookmark-meta))) - -;; finds nodes with unique id and if these are not unique anymore, replace the numbering id with a duplicate numbering definition -(defn fix-numbering-reset [xml-tree] - (let [found (atom #{}) - last-val (atom nil) - edit-attrs (fn [attrs] - (let [old-id (ooxml/val attrs) - unq-id (:stencil.model/unique attrs) - new-id old-id] ;; TODO - ;; ha az unique id benne van a found set-ben - ;; akkor uj value-t szamolunk es beallitjuk a last-val-t. - ;; egyebkent ha a last-val nem ugyanaz mint a mostani, akkor beallitjuk a last val-t (?) - (-> attrs - (dissoc :stencil.model/unique) - (assoc ooxml/val new-id))))] - ;; find nodes with unique id in attrs - (dfs-walk-xml-node xml-tree (fn [n] (and (map? n) (:stencil.model/unique (:attrs n)))) - (fn [n] (zip/edit n update :attrs edit-attrs))))) diff --git a/src/stencil/postprocess/numberings.clj b/src/stencil/postprocess/numberings.clj new file mode 100644 index 00000000..bea3c405 --- /dev/null +++ b/src/stencil/postprocess/numberings.clj @@ -0,0 +1,30 @@ +(ns stencil.postprocess.numberings + (:require [stencil.util :refer :all] + [stencil.ooxml :as ooxml] + [stencil.model.numbering :as numbering] + [stencil.log :as log] + [clojure.zip :as zip])) + +(defn- get-new-id [numbering-id trace] + (log/debug "Getting new list for old {} trace {}" numbering-id trace) + (if (empty? trace) + numbering-id + (numbering/copy-numbering! numbering-id))) + +(defn- lookup [get-new-id element] + (get-new-id (ooxml/val (:attrs element)) + (take-last (:stencil.cleanup/depth (:attrs element)) + (:stencil.eval/trace (:attrs element))))) + +(defn- fix-one [numbering lookup] + (-> numbering + (update :attrs dissoc :stencil.cleanup/depth :stencil.eval/trace) + (update :attrs assoc ooxml/val (lookup numbering)))) + +(defn fix-numberings [xml-tree] + (let [lookup (partial lookup (memoize get-new-id))] + (dfs-walk-xml-node + xml-tree + (fn [e] (= ooxml/attr-numId (:tag e))) + (fn [e] (zip/edit e fix-one lookup))))) + diff --git a/src/stencil/tree_postprocess.clj b/src/stencil/tree_postprocess.clj index 2cb5f7b8..fa7f875b 100644 --- a/src/stencil/tree_postprocess.clj +++ b/src/stencil/tree_postprocess.clj @@ -6,6 +6,7 @@ [stencil.postprocess.images :refer :all] [stencil.postprocess.list-ref :refer :all] [stencil.postprocess.fragments :refer :all] + [stencil.postprocess.numberings :refer [fix-numberings]] [stencil.postprocess.html :refer :all])) ;; calls postprocess @@ -26,7 +27,7 @@ #'fix-list-dirty-refs - #'fix-numbering-reset + #'fix-numberings #'replace-images diff --git a/test/stencil/model/numbering_test.clj b/test/stencil/model/numbering_test.clj index 88cc9ded..be925502 100644 --- a/test/stencil/model/numbering_test.clj +++ b/test/stencil/model/numbering_test.clj @@ -10,11 +10,12 @@ {:tag tag, :attrs attrs, :content (mapv hiccup children)})) (deftest test-style-for-def-empty - (binding [*numbering* {:parsed (prepare-numbering-xml {:tag :numbering :content []})}] + (binding [*numbering* (atom {:parsed (prepare-numbering-xml {:tag :numbering :content []})})] (is (= nil (style-def-for "id-1" 2))))) (deftest test-style-for-def-with-abstract (binding [*numbering* + (atom {:parsed (prepare-numbering-xml (hiccup @@ -27,13 +28,14 @@ [:lvlText {ooxml/val ""}] [:lvlJc {ooxml/val "start"}]]] [ooxml/tag-num {ooxml/attr-numId "id-1"} - [ooxml/xml-abstract-num-id {ooxml/val "a1"}]]]))}] + [ooxml/xml-abstract-num-id {ooxml/val "a1"}]]]))})] (is (= {:lvl-text "", :num-fmt "none", :start 1} (style-def-for "id-1" 2))))) (deftest test-style-for-def (binding [*numbering* + (atom {:parsed (prepare-numbering-xml (hiccup @@ -44,6 +46,6 @@ [:numFmt {ooxml/val "none"}] [:suff {ooxml/val "nothing"}] [:lvlText {ooxml/val ""}] - [:lvlJc {ooxml/val "start"}]]]]))}] + [:lvlJc {ooxml/val "start"}]]]]))})] (is (= {:lvl-text "", :num-fmt "none", :start 1} (style-def-for "id-1" 2))))) From ddbda23e0fb67065940a361c163a3c5c3f3929af Mon Sep 17 00:00:00 2001 From: erdos Date: Sun, 25 Dec 2022 19:48:55 +0100 Subject: [PATCH 5/8] wip --- src/stencil/model/numbering.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/stencil/model/numbering.clj b/src/stencil/model/numbering.clj index b264a0df..ec43ae5e 100644 --- a/src/stencil/model/numbering.clj +++ b/src/stencil/model/numbering.clj @@ -88,7 +88,10 @@ ;; returns new id for the numbering copied from old-id (defn copy-numbering! [old-id] (let [old-elem (find-first (fn [e] (-> e :attrs ooxml/attr-numId (= old-id))) (:content (:parsed @*numbering*))) - new-id #_(name (gensym "xx")) (str (int (* 1000 (Math/random)))) + max-num-id (apply max (keep (comp ->int ooxml/attr-numId :attrs) + (:content (:parsed @*numbering*)))) + new-id (str (inc max-num-id)) +; new-id #_(name (gensym "xx")) (str (int (* 1000 (Math/random)))) new-elem (assoc-in old-elem [:attrs ooxml/attr-numId] new-id)] (assert old-elem) (swap! *numbering* update :parsed update :content concat [new-elem]) From 39ef289fb2f73fb3903f802c3e6c5c524f1f9cce Mon Sep 17 00:00:00 2001 From: erdos Date: Sun, 25 Dec 2022 22:05:45 +0100 Subject: [PATCH 6/8] wip --- src/stencil/model/numbering.clj | 32 ++++++++++++++++++++++++++------ src/stencil/ooxml.clj | 3 +++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/stencil/model/numbering.clj b/src/stencil/model/numbering.clj index ec43ae5e..6932e540 100644 --- a/src/stencil/model/numbering.clj +++ b/src/stencil/model/numbering.clj @@ -85,17 +85,37 @@ (get-id-style-xml id lvl) (xml-lvl-parse))) -;; returns new id for the numbering copied from old-id -(defn copy-numbering! [old-id] - (let [old-elem (find-first (fn [e] (-> e :attrs ooxml/attr-numId (= old-id))) (:content (:parsed @*numbering*))) + +(defn- tag-lvl-start-override [lvl start] + {:tag ooxml/lvl-override + :attrs {ooxml/attr-ilvl lvl} + :content [{:tag ooxml/start-override :attrs {ooxml/val start}}]}) + + +(defn copy-numbering! + "Creates a copy of the numbering definition an returns the new id for it." + [old-id] + (let [old-elem (find-first (fn [e] (-> e :attrs ooxml/attr-numId (= old-id))) + (:content (:parsed @*numbering*))) + abstract-num-id (some (fn [e] + (when (= ooxml/xml-abstract-num-id (:tag e)) + (-> e :attrs ooxml/val))) + (:content old-elem)) max-num-id (apply max (keep (comp ->int ooxml/attr-numId :attrs) (:content (:parsed @*numbering*)))) new-id (str (inc max-num-id)) -; new-id #_(name (gensym "xx")) (str (int (* 1000 (Math/random)))) - new-elem (assoc-in old-elem [:attrs ooxml/attr-numId] new-id)] + new-elem (assoc-in old-elem [:attrs ooxml/attr-numId] new-id) + new-elem (update new-elem :content concat + (for [abstract (:content (:parsed @*numbering*)) + :when (= abstract-num-id (-> abstract :attrs ooxml/xml-abstract-num-id)) + lvl (:content abstract) + :when (= (:tag lvl) ooxml/tag-lvl) + start (:content lvl) + :when (= "start" (name (:tag start)))] + (tag-lvl-start-override (-> lvl :attrs ooxml/attr-ilvl) (-> start :attrs ooxml/val))))] (assert old-elem) (swap! *numbering* update :parsed update :content concat [new-elem]) (swap! *numbering* dissoc :source-file) (swap! *numbering* (fn [numbering] (assoc numbering :result {:writer (stencil.model.common/->xml-writer (:parsed numbering))}))) - new-id)) \ No newline at end of file + new-id)) diff --git a/src/stencil/ooxml.clj b/src/stencil/ooxml.clj index 171680ef..8519baeb 100644 --- a/src/stencil/ooxml.clj +++ b/src/stencil/ooxml.clj @@ -68,6 +68,9 @@ (def attr-ilvl :xmlns.http%3A%2F%2Fschemas.openxmlformats.org%2Fwordprocessingml%2F2006%2Fmain/ilvl) +(def lvl-override :xmlns.http%3A%2F%2Fschemas.openxmlformats.org%2Fwordprocessingml%2F2006%2Fmain/lvlOverride) +(def start-override :xmlns.http%3A%2F%2Fschemas.openxmlformats.org%2Fwordprocessingml%2F2006%2Fmain/startOverride) + (def default-aliases {;default namespace aliases from a LibreOffice 6.4 OOXML Text document "http://schemas.openxmlformats.org/markup-compatibility/2006" "mc" From 23d0a3504f7fcd07362e1a4b24bcbf5044d8f9ab Mon Sep 17 00:00:00 2001 From: erdos Date: Sun, 25 Dec 2022 22:09:16 +0100 Subject: [PATCH 7/8] wip --- src/stencil/cleanup.clj | 17 ----------------- src/stencil/model.clj | 10 ---------- 2 files changed, 27 deletions(-) diff --git a/src/stencil/cleanup.clj b/src/stencil/cleanup.clj index d3408a2d..1525a14b 100644 --- a/src/stencil/cleanup.clj +++ b/src/stencil/cleanup.clj @@ -211,22 +211,6 @@ :when (= :cmd/include (:cmd item))] (:name item)))) -;; Calculate for each numbering how many for loops deep it is. -(defn- numid->depth [ast] - (let [cache (volatile! {})] - ;; visit all nodes and set up a cache. - (doseq [e ast] - ((fn f [d node] - (assert (map? node)) - (when (= ooxml/attr-numId (:open+close node)) - (vswap! cache update (ooxml/val (:attrs node)) (fnil conj #{}) d)) - (let [d (if (= :for (:cmd node)) (cons (gensym "x") d) d)] - (doseq [block (::blocks node) - b (::children block)] - (f d b)))) () e)) - (into {} (for [[k v] @cache] - [k (count (take-while true? (apply map = (map reverse v))))])))) - ;; add ::depth to ooxml/numId elements (defn- ast-numbering-depths [ast] (let [numid->paths (volatile! {}) @@ -263,7 +247,6 @@ executable (mapv control-ast-normalize (annotate-environments ast))] {:variables (find-variables ast) :fragments (find-fragments ast) - :num-depth (numid->depth ast) :dynamic? (boolean (some :cmd executable)) :executable executable})) diff --git a/src/stencil/model.clj b/src/stencil/model.clj index 7db40d57..8f0ae453 100644 --- a/src/stencil/model.clj +++ b/src/stencil/model.clj @@ -53,19 +53,9 @@ {:source-file cts ::path (.getName cts)}) -#_ -(defn- add-unique-index - "Annotates some elements with an unique id. - These elements need special care when rendering duplicates them. - For example, numberings need to be reset to start from 1 again." - [elem] - (when (= (:open+close elem) ooxml/attr-numId) - (update elem :attrs assoc ::unique (gensym "uniq")))) - (defn ->exec [xml-streamable] (with-open [stream (io/input-stream xml-streamable)] (-> (merger/parse-to-tokens-seq stream) - ;(->> (map (some-fn add-unique-index identity))) (cleanup/process) (select-keys [:variables :dynamic? :executable :fragments])))) From 16e3ae4cabdddf58cae6f5720dc26b1d6b9cd91e Mon Sep 17 00:00:00 2001 From: erdos Date: Mon, 26 Dec 2022 13:18:41 +0100 Subject: [PATCH 8/8] cleanup --- src/stencil/cleanup.clj | 104 ++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 53 deletions(-) diff --git a/src/stencil/cleanup.clj b/src/stencil/cleanup.clj index 1525a14b..f39ba3d2 100644 --- a/src/stencil/cleanup.clj +++ b/src/stencil/cleanup.clj @@ -8,8 +8,8 @@ valid XML String -> tokens -> Annotated Control AST -> Normalized Control AST -> Evaled AST -> Hiccup or valid XML String " (:require [stencil.util :refer [mod-stack-top-conj mod-stack-top-last parsing-exception stacks-difference-key]] - [stencil.ooxml :as ooxml] - [stencil.types :refer [open-tag close-tag]])) + [stencil.types :refer [open-tag close-tag]] + [stencil.ooxml :as ooxml])) (set! *warn-on-reflection* true) @@ -78,24 +78,24 @@ (assert (sequential? control-ast)) (let [stack (volatile! ())] (mapv (partial nested-tokens-fmap-postwalk - (fn before-cmd-block [block] - (assoc block ::before @stack)) - - (fn after-cmd-block [block] - (let [stack-before (::before block) - [a b] (stacks-difference-key :open stack-before @stack)] - (assoc block ::before a ::after b))) - - (fn child [item] - (cond - (:open item) - (vswap! stack conj item) - - (:close item) - (if (= (:close item) (:open (first @stack))) - (vswap! stack next) - (throw (ex-info "Unexpected stack state" {:stack @stack, :item item})))) - item)) + (fn before-cmd-block [block] + (assoc block ::before @stack)) + + (fn after-cmd-block [block] + (let [stack-before (::before block) + [a b] (stacks-difference-key :open stack-before @stack)] + (assoc block ::before a ::after b))) + + (fn child [item] + (cond + (:open item) + (vswap! stack conj item) + + (:close item) + (if (= (:close item) (:open (first @stack))) + (vswap! stack next) + (throw (ex-info "Unexpected stack state" {:stack @stack, :item item})))) + item)) control-ast))) (defn stack-revert-close @@ -167,43 +167,42 @@ ;; amikor van benne blocks ;; mapping: {Sym -> Str} (letfn [(resolve-sym [mapping s] - (assert (map? mapping)) - (assert (symbol? s)) - (mapping s (name s))) + (assert (map? mapping)) + (assert (symbol? s)) + (mapping s (name s))) (expr [mapping e] - (cond (symbol? e) [(resolve-sym mapping e)] - (not (sequential? e)) nil - (= :fncall (first e)) (mapcat (partial expr mapping) (nnext e)) - (= :get (first e)) (let [[ss rest] (split-with string? (nnext e))] - (cons - (reduce (fn [root item] (str root "." item)) - (resolve-sym mapping (second e)) - ss) - (mapcat (partial expr mapping) rest))) - :else (mapcat (partial expr mapping) (next e)))) + (cond (symbol? e) [(resolve-sym mapping e)] + (not (sequential? e)) nil + (= :fncall (first e)) (mapcat (partial expr mapping) (nnext e)) + (= :get (first e)) (let [[ss rest] (split-with string? (nnext e))] + (cons + (reduce (fn [root item] (str root "." item)) + (resolve-sym mapping (second e)) + ss) + (mapcat (partial expr mapping) rest))) + :else (mapcat (partial expr mapping) (next e)))) (maybe-variable [mapping e] - (cond (symbol? e) - (resolve-sym mapping e) - (and (sequential? e) (= :get (first e)) (symbol? (second e)) (every? string? (nnext e))) - (reduce (fn [a b] (str a "." b)) (resolve-sym mapping (second e)) (nnext e)))) + (cond (symbol? e) + (resolve-sym mapping e) + (and (sequential? e) (= :get (first e)) (symbol? (second e)) (every? string? (nnext e))) + (reduce (fn [a b] (str a "." b)) (resolve-sym mapping (second e)) (nnext e)))) (collect [m xs] (mapcat (partial collect-1 m) xs)) (collect-1 [mapping x] - (case (:cmd x) - :cmd/echo (expr mapping (:expression x)) - - :if (concat (expr mapping (:condition x)) - (collect mapping (apply concat (::blocks x)))) - - :for (let [variable (maybe-variable mapping (:expression x)) - exprs (expr mapping (:expression x)) - mapping (if variable - (assoc mapping (:variable x) (str variable "[]")) - mapping)] - (concat exprs (collect mapping (apply concat (::blocks x))))) - []))] + (case (:cmd x) + :cmd/echo (expr mapping (:expression x)) + + :if (concat (expr mapping (:condition x)) + (collect mapping (apply concat (::blocks x)))) + + :for (let [variable (maybe-variable mapping (:expression x)) + exprs (expr mapping (:expression x)) + mapping (if variable + (assoc mapping (:variable x) (str variable "[]")) + mapping)] + (concat exprs (collect mapping (apply concat (::blocks x))))) + []))] (distinct (collect {} control-ast)))) -;; TODO: does it work? (defn- find-fragments [control-ast] ;; returns a set of fragment names use in this document (set (for [item (tree-seq map? (comp flatten ::blocks) {::blocks [control-ast]}) @@ -249,5 +248,4 @@ :fragments (find-fragments ast) :dynamic? (boolean (some :cmd executable)) :executable executable})) - -:OK +:OK \ No newline at end of file