Skip to content

Commit

Permalink
Revamp doc generation
Browse files Browse the repository at this point in the history
Auto generated docs are **NO LONGER CHECKED IN**, only placeholders.

To generate them, e.g. before exporting docs, run hack/generate-docs.sh.

hack/verify-generated-docs.sh ensures that generated docs are merely the
placeholder text.

hack/update-generated-docs.sh puts the placeholder text in the proper
places.

The old munge behavior is moved into hack/{update|verify}-munge-docs.sh.
  • Loading branch information
lavalamp committed Jun 9, 2016
1 parent 34dcea3 commit 8faa886
Show file tree
Hide file tree
Showing 8 changed files with 244 additions and 102 deletions.
1 change: 0 additions & 1 deletion build/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -483,7 +483,6 @@ function kube::build::source_targets() {
third_party
vendor
contrib/mesos
.generated_docs
)
if [ -n "${KUBERNETES_CONTRIB:-}" ]; then
for contrib in "${KUBERNETES_CONTRIB}"; do
Expand Down
3 changes: 3 additions & 0 deletions hack/autogenerated_placeholder.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
This file is autogenerated, but we've stopped checking such files into the
repository to reduce the need for rebases. Please run hack/generate-docs.sh to
populate this file.
52 changes: 52 additions & 0 deletions hack/generate-docs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#!/bin/bash

# Copyright 2014 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is not intended to be run automatically. It is meant to be run
# immediately before exporting docs. We do not want to check these documents in
# by default.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"

kube::golang::setup_env

"${KUBE_ROOT}/hack/build-go.sh" \
cmd/gendocs \
cmd/genkubedocs \
cmd/genman \
cmd/genyaml \
federation/cmd/genfeddocs

kube::util::ensure-temp-dir

kube::util::gen-docs "${KUBE_TEMP}"

# remove all of the old docs
kube::util::remove-gen-docs

# Copy fresh docs into the repo.
# the shopt is so that we get .generated_docs from the glob.
shopt -s dotglob
cp -af "${KUBE_TEMP}"/* "${KUBE_ROOT}"
shopt -u dotglob

echo "Generated docs have been placed in the repository tree. Running hack/update-munge-docs.sh."

"${KUBE_ROOT}/hack/update-munge-docs.sh"
45 changes: 26 additions & 19 deletions hack/lib/util.sh
Original file line number Diff line number Diff line change
Expand Up @@ -208,27 +208,34 @@ kube::util::gen-docs() {
touch .generated_docs
find . -type f | cut -sd / -f 2- | LC_ALL=C sort > .generated_docs
popd > /dev/null
}

while read file; do
# Copy out of KUBE_ROOT if we didn't really change anything
if [[ -e "${dest}/${file}" && -e "${KUBE_ROOT}/${file}" ]]; then
# Filter all munges from original content.
original=$(cat "${KUBE_ROOT}/${file}" | sed '/^<!-- BEGIN MUNGE:.*/,/^<!-- END MUNGE:.*/d')
generated=$(cat "${dest}/${file}")

# Filter out meaningless lines with timestamps
original=$(echo "${original}" | grep -v "Auto generated by spf13/cobra" || :)
generated=$(echo "${generated}" | grep -v "Auto generated by spf13/cobra" || :)

# By now, the contents should be normalized and stripped of any
# auto-managed content. We also ignore whitespace here because of
# markdown strictness fixups later in the pipeline.
if diff -Bw >/dev/null <(echo "${original}") <(echo "${generated}"); then
# actual contents same, overwrite generated with original.
cp "${KUBE_ROOT}/${file}" "${dest}/${file}"
# Puts a placeholder for every generated doc. This makes the link checker work.
kube::util::set-placeholder-gen-docs() {
local list_file="${KUBE_ROOT}/.generated_docs"
if [ -e ${list_file} ]; then
# remove all of the old docs; we don't want to check them in.
while read file; do
if [[ "${list_file}" != "${KUBE_ROOT}/${file}" ]]; then
cp "${KUBE_ROOT}/hack/autogenerated_placeholder.txt" "${KUBE_ROOT}/${file}"
fi
fi
done <"${KUBE_ROOT}/.generated_docs"
done <"${list_file}"
# The .generated_docs file lists itself, so we don't need to explicitly
# delete it.
fi
}

# Removes previously generated docs-- we don't want to check them in. $KUBE_ROOT
# must be set.
kube::util::remove-gen-docs() {
if [ -e "${KUBE_ROOT}/.generated_docs" ]; then
# remove all of the old docs; we don't want to check them in.
while read file; do
rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
done <"${KUBE_ROOT}/.generated_docs"
# The .generated_docs file lists itself, so we don't need to explicitly
# delete it.
fi
}

# Takes a path $1 to traverse for md files to append the ga-beacon tracking
Expand Down
46 changes: 10 additions & 36 deletions hack/update-generated-docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is not intended to be run automatically. It is meant to be run
# immediately before exporting docs. We do not want to check these documents in
# by default.

set -o errexit
set -o nounset
set -o pipefail
Expand All @@ -28,54 +32,24 @@ kube::golang::setup_env
cmd/genkubedocs \
cmd/genman \
cmd/genyaml \
cmd/mungedocs \
federation/cmd/genfeddocs

kube::util::ensure-temp-dir

kube::util::gen-docs "${KUBE_TEMP}"

# remove all of the old docs
while read file; do
rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
done <"${KUBE_ROOT}/.generated_docs"
kube::util::remove-gen-docs

# Copy fresh docs into the repo.
# the shopt is so that we get .generated_docs from the glob.
shopt -s dotglob
cp -af "${KUBE_TEMP}"/* "${KUBE_ROOT}"
shopt -u dotglob

kube::util::gen-analytics "${KUBE_ROOT}"

mungedocs=$(kube::util::find-binary "mungedocs")
"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${KUBE_ROOT}/docs/" && ret=0 || ret=$?
if [[ $ret -eq 1 ]]; then
echo "${KUBE_ROOT}/docs/ requires manual changes. See preceding errors."
exit 1
elif [[ $ret -gt 1 ]]; then
echo "Error running mungedocs."
exit 1
fi

"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${KUBE_ROOT}/examples/" && ret=0 || ret=$?
if [[ $ret -eq 1 ]]; then
echo "${KUBE_ROOT}/examples/ requires manual changes. See preceding errors."
exit 1
elif [[ $ret -gt 1 ]]; then
echo "Error running mungedocs."
exit 1
fi
# Replace with placeholder docs
kube::util::set-placeholder-gen-docs

"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" \
"--skip-munges=unversioned-warning,analytics" \
"--norecurse" \
"--root-dir=${KUBE_ROOT}/" && ret=0 || ret=$?
if [[ $ret -eq 1 ]]; then
echo "${KUBE_ROOT}/ requires manual changes. See preceding errors."
exit 1
elif [[ $ret -gt 1 ]]; then
echo "Error running mungedocs."
exit 1
fi
echo "Generated docs have been placed in the repository tree. Running hack/update-munge-docs.sh."

# ex: ts=2 sw=2 et filetype=sh
"${KUBE_ROOT}/hack/update-munge-docs.sh"
64 changes: 64 additions & 0 deletions hack/update-munge-docs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/bin/bash

# Copyright 2014 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"

kube::golang::setup_env

"${KUBE_ROOT}/hack/build-go.sh" \
cmd/mungedocs

kube::util::ensure-temp-dir

kube::util::gen-analytics "${KUBE_ROOT}"

mungedocs=$(kube::util::find-binary "mungedocs")
"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${KUBE_ROOT}/docs/" && ret=0 || ret=$?
if [[ $ret -eq 1 ]]; then
echo "${KUBE_ROOT}/docs/ requires manual changes. See preceding errors."
exit 1
elif [[ $ret -gt 1 ]]; then
echo "Error running mungedocs."
exit 1
fi

"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${KUBE_ROOT}/examples/" && ret=0 || ret=$?
if [[ $ret -eq 1 ]]; then
echo "${KUBE_ROOT}/examples/ requires manual changes. See preceding errors."
exit 1
elif [[ $ret -gt 1 ]]; then
echo "Error running mungedocs."
exit 1
fi

"${mungedocs}" "--upstream=${KUBE_GIT_UPSTREAM}" \
"--skip-munges=unversioned-warning,analytics" \
"--norecurse" \
"--root-dir=${KUBE_ROOT}/" && ret=0 || ret=$?
if [[ $ret -eq 1 ]]; then
echo "${KUBE_ROOT}/ requires manual changes. See preceding errors."
exit 1
elif [[ $ret -gt 1 ]]; then
echo "Error running mungedocs."
exit 1
fi

# ex: ts=2 sw=2 et filetype=sh
64 changes: 18 additions & 46 deletions hack/verify-generated-docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,57 +28,29 @@ kube::golang::setup_env
cmd/genkubedocs \
cmd/genman \
cmd/genyaml \
cmd/mungedocs

# Find binary
mungedocs=$(kube::util::find-binary "mungedocs")

DOCROOT="${KUBE_ROOT}/docs/"
EXAMPLEROOT="${KUBE_ROOT}/examples/"

# mungedocs --verify can (and should) be run on the real docs, otherwise their
# links will be distorted. --verify means that it will not make changes.
# --verbose gives us output we can use for a diff.
"${mungedocs}" "--verify=true" "--verbose=true" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${DOCROOT}" && ret=0 || ret=$?
if [[ $ret -eq 1 ]]; then
echo "${DOCROOT} is out of date. Please run hack/update-generated-docs.sh"
exit 1
fi
if [[ $ret -gt 1 ]]; then
echo "Error running mungedocs"
exit 1
fi

"${mungedocs}" "--verify=true" "--verbose=true" "--upstream=${KUBE_GIT_UPSTREAM}" "--root-dir=${EXAMPLEROOT}" && ret=0 || ret=$?
if [[ $ret -eq 1 ]]; then
echo "${EXAMPLEROOT} is out of date. Please run hack/update-generated-docs.sh"
exit 1
fi
if [[ $ret -gt 1 ]]; then
echo "Error running mungedocs"
exit 1
fi
federation/cmd/genfeddocs

kube::util::ensure-temp-dir

kube::util::gen-docs "${KUBE_TEMP}"
diff -Naup "${KUBE_TEMP}/.generated_docs" "${KUBE_ROOT}/.generated_docs" || ret=1 || true
while read file; do
diff -Naup "${KUBE_TEMP}/${file}" "${KUBE_ROOT}/${file}" || ret=1 || true
done <"${KUBE_TEMP}/.generated_docs"

needsanalytics=($(kube::util::gen-analytics "${KUBE_ROOT}" 1))
if [[ ${#needsanalytics[@]} -ne 0 ]]; then
echo -e "Some md files are missing ga-beacon analytics link:"
printf '%s\n' "${needsanalytics[@]}"
ret=1
fi
if [[ $ret -eq 0 ]]
then
echo "Generated docs are up to date."
else
echo "Generated docs are out of date. Please run hack/update-generated-docs.sh"
# Verify the list matches the expected list (diff should be empty)
if [[ "$(diff ${KUBE_ROOT}/.generated_docs ${KUBE_TEMP}/.generated_docs)" != "" ]]; then
echo "List of generated docs doesn't match a freshly built list. Please run hack/update-generated-docs.sh"
exit 1
fi

# ex: ts=2 sw=2 et filetype=sh
# Verify the files in the repo all contain the boilerplate instead of the actual
# content.
while read file; do
# Ignore .generated_docs-- it should not have the boilerplate!
[[ "${file}" == ".generated_docs" ]] && continue

# Search for "hack/generate-docs.sh" as a proxy for the boilerplate content,
# since the munger adds a bunch of other stuff.
if [[ "$(grep "hack/generate-docs.sh" "${KUBE_ROOT}/${file}")" == "" ]]; then
echo "${file} doesn't seem to have the correct boilerplate content for an autogenerated file."
echo "Please run hack/update-generated-docs.sh"
exit 1
fi
done <"${KUBE_ROOT}/.generated_docs"
Loading

0 comments on commit 8faa886

Please sign in to comment.