From a42b22f862e1143f14a5ba2cb79b008eec74f193 Mon Sep 17 00:00:00 2001 From: Franco Fichtner Date: Mon, 16 Sep 2024 10:50:04 +0200 Subject: [PATCH 1/9] scripts: add a helper for port-based MFC workflow --- scripts/port_dir_mfc.sh | 78 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 scripts/port_dir_mfc.sh diff --git a/scripts/port_dir_mfc.sh b/scripts/port_dir_mfc.sh new file mode 100755 index 00000000..24ade3da --- /dev/null +++ b/scripts/port_dir_mfc.sh @@ -0,0 +1,78 @@ +#!/bin/sh + +# Copyright (c) 2024 Franco Fichtner +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +set -e + +DIR=${1:-.} +BRANCH=${2:-master} +FROM=FreeBSD + +if git diff --quiet ${BRANCH} ${DIR}; then + echo ">>> Cherry-pick already complete." + exit 0 +fi + +echo -n ">>> Run a git-cherry-pick or raw merge? [r/G]: " + +read YN < /dev/tty +case ${YN} in +[rR]) + git diff -R ${BRANCH} ${DIR} | git apply + git add ${DIR} + git commit -m \ +"${DIR}: sync with upstream + +Taken from: ${FROM}" + exit 0 + ;; +*) + # FALLTHROUGH + ;; +esac + +COMMITS= + +for HASH in $(git log --oneline ${BRANCH} ${DIR} | awk '{ print $1 }'); do + if git diff --quiet ${HASH} ${DIR}; then + # found no more changes + break + fi + + # reverse commit order for cherry-pick + COMMITS="${HASH} ${COMMITS}" +done + +for COMMIT in ${COMMITS}; do + git cherry-pick ${COMMIT} || git cherry-pick --skip +done + +if ! git diff --quiet ${BRANCH} ${DIR}; then + echo ">>> Cherry-pick failed due to non-emtpy diff." >&2 + exit 1 +fi + +echo ">>> Cherry-pick finished successfully." From b6cd7fbabe2dd220f1906762823e6b4cd143aef3 Mon Sep 17 00:00:00 2001 From: Franco Fichtner Date: Mon, 16 Sep 2024 20:46:11 +0200 Subject: [PATCH 2/9] build/sync: turn the port_dir_mfc script into a 'sync' step --- Makefile | 4 +- README.md | 11 ++- build/skim.sh | 2 +- scripts/port_dir_mfc.sh => build/sync.sh | 89 +++++++++++++----------- 4 files changed, 63 insertions(+), 43 deletions(-) rename scripts/port_dir_mfc.sh => build/sync.sh (56%) diff --git a/Makefile b/Makefile index 83f67662..52ac4d47 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,8 @@ STEPS= audit arm base boot chroot clean clone compress confirm \ connect core distfiles download dvd fingerprint info \ kernel list make.conf nano options packages plugins ports \ - prefetch print rebase release rename serial sign \ - skim test update upload verify vga vm xtools + prefetch print rebase release rename serial sign skim \ + sync test update upload verify vga vm xtools SCRIPTS= custom distribution factory hotfix nightly watch .PHONY: ${STEPS} ${SCRIPTS} diff --git a/README.md b/README.md index a9b47808..d331c90a 100644 --- a/README.md +++ b/README.md @@ -371,7 +371,7 @@ Available clean options are: * vm: remove vm image * xtools: remove xtools set -How the port tree is synced with its upstream repository +How the port tree is updated via its upstream repository -------------------------------------------------------- The ports tree has a few of our modifications and is sometimes a @@ -422,6 +422,15 @@ for standalone use on the host use: # make make.conf +Syncing a ports branch from PORTSDIR +------------------------------------ + +When maintaining branches the master branch holds updates that +we want to cherry-pick to another branch. To ease the process +the sync step can deal with the complexity involved: + + # make sync-category/port[,category/port[,...]] + Reading and modifying version numbers of build sets and images -------------------------------------------------------------- diff --git a/build/skim.sh b/build/skim.sh index ec4b95a2..978bd122 100644 --- a/build/skim.sh +++ b/build/skim.sh @@ -27,8 +27,8 @@ set -e -SELF=skim FROM=FreeBSD +SELF=skim . ./common.sh diff --git a/scripts/port_dir_mfc.sh b/build/sync.sh similarity index 56% rename from scripts/port_dir_mfc.sh rename to build/sync.sh index 24ade3da..61d898f5 100755 --- a/scripts/port_dir_mfc.sh +++ b/build/sync.sh @@ -27,52 +27,63 @@ set -e -DIR=${1:-.} -BRANCH=${2:-master} FROM=FreeBSD +SELF=sync -if git diff --quiet ${BRANCH} ${DIR}; then - echo ">>> Cherry-pick already complete." - exit 0 -fi +. ./common.sh -echo -n ">>> Run a git-cherry-pick or raw merge? [r/G]: " +GIT="git -C ${PORTSDIR}" -read YN < /dev/tty -case ${YN} in -[rR]) - git diff -R ${BRANCH} ${DIR} | git apply - git add ${DIR} - git commit -m \ -"${DIR}: sync with upstream +for ARG in ${@}; do + # ARG should be "category/name" but not strictly checked -Taken from: ${FROM}" - exit 0 - ;; -*) - # FALLTHROUGH - ;; -esac - -COMMITS= - -for HASH in $(git log --oneline ${BRANCH} ${DIR} | awk '{ print $1 }'); do - if git diff --quiet ${HASH} ${DIR}; then - # found no more changes - break + if [ ! -d ${PORTSDIR}/${ARG} ]; then + echo ">>> Sync did not find the port ${ARG}" >&2 + exit 1 fi - # reverse commit order for cherry-pick - COMMITS="${HASH} ${COMMITS}" -done + if ${GIT} diff --quiet ${PORTSBRANCH} ${ARG}; then + echo ">>> Sync already complete for ${ARG}" + continue + fi -for COMMIT in ${COMMITS}; do - git cherry-pick ${COMMIT} || git cherry-pick --skip -done -if ! git diff --quiet ${BRANCH} ${DIR}; then - echo ">>> Cherry-pick failed due to non-emtpy diff." >&2 - exit 1 -fi + COMMITS= + + for HASH in $(${GIT} log --oneline ${PORTSBRANCH} ${ARG} | \ + awk '{ print $1 }'); do + if ${GIT} diff --quiet ${HASH} ${ARG}; then + # found no more changes + break + fi + + # reverse commit order for cherry-pick + COMMITS="${HASH} ${COMMITS}" + done + + FAILED= + + for COMMIT in ${COMMITS}; do + if ! (${GIT} cherry-pick ${COMMIT} || \ + ${GIT} cherry-pick --skip); then + FAILED=yes + break + fi + done -echo ">>> Cherry-pick finished successfully." + if [ -n "${FAILED}" ]; then + ${GIT} diff -R ${PORTSBRANCH} ${ARG} | ${GIT} apply + ${GIT} add ${ARG} + ${GIT} commit -m \ +"${ARG}: sync with upstream + +Taken from: ${FROM}" + fi + + if ! ${GIT} diff --quiet ${PORTSBRANCH} ${ARG}; then + echo ">>> Sync failed due to non-emtpy diff for ${ARG}" >&2 + exit 1 + fi + + echo ">>> Sync succeeded for ${ARG}" +done From bee2bccfd062a72698b355d3f42d4d6afc5a7ee2 Mon Sep 17 00:00:00 2001 From: Franco Fichtner Date: Mon, 16 Sep 2024 20:51:32 +0200 Subject: [PATCH 3/9] README: small tweak --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d331c90a..6df4461e 100644 --- a/README.md +++ b/README.md @@ -422,8 +422,8 @@ for standalone use on the host use: # make make.conf -Syncing a ports branch from PORTSDIR ------------------------------------- +Syncing a ports branch +---------------------- When maintaining branches the master branch holds updates that we want to cherry-pick to another branch. To ease the process From c9b0661d7b257945dd0f8a7e4ba6e9067f46f96a Mon Sep 17 00:00:00 2001 From: Monviech Date: Mon, 16 Sep 2024 16:18:47 +0000 Subject: [PATCH 4/9] www/caddy-custom: Add ratelimit module. --- config/24.7/make.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/24.7/make.conf b/config/24.7/make.conf index 52b6fdfb..33783f30 100644 --- a/config/24.7/make.conf +++ b/config/24.7/make.conf @@ -97,6 +97,7 @@ www_webgrind_SET= CALLGRAPH CADDY_CUSTOM_PLUGINS= github.com/caddyserver/ntlm-transport@e0c1e46a30093fa243d06a83964da5573ee6a51f \ github.com/mholt/caddy-dynamicdns@d8dab1bbf3fc592032f71dacc14510475b4e3e9a \ github.com/mholt/caddy-l4@e23bce071de6534a33e7a0c0838b111e11c59f54 \ + github.com/mholt/caddy-ratelimit@12435ecef5dbb1b137eb68002b85d775a9d5cdb2 \ github.com/caddy-dns/cloudflare@89f16b99c18ef49c8bb470a82f895bce01cbaece \ github.com/caddy-dns/route53@5e0037b52d9b6dbe8ef47d5d3fdf42d7f87ebf79 \ github.com/caddy-dns/duckdns@77870e12bac552ceb76917d82ced6db84b958c1f \ From 7fa789ea6c40864a384d534f00869573a97ef857 Mon Sep 17 00:00:00 2001 From: Monviech Date: Sun, 22 Sep 2024 14:31:46 +0000 Subject: [PATCH 5/9] www/caddy-custom: Update dependencies --- config/24.7/make.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/24.7/make.conf b/config/24.7/make.conf index 33783f30..8eff9fde 100644 --- a/config/24.7/make.conf +++ b/config/24.7/make.conf @@ -96,22 +96,22 @@ www_webgrind_SET= CALLGRAPH # for www/caddy-custom CADDY_CUSTOM_PLUGINS= github.com/caddyserver/ntlm-transport@e0c1e46a30093fa243d06a83964da5573ee6a51f \ github.com/mholt/caddy-dynamicdns@d8dab1bbf3fc592032f71dacc14510475b4e3e9a \ - github.com/mholt/caddy-l4@e23bce071de6534a33e7a0c0838b111e11c59f54 \ + github.com/mholt/caddy-l4@4f012d4517cf65b3a2da1308ec6e770c0cf0b656 \ github.com/mholt/caddy-ratelimit@12435ecef5dbb1b137eb68002b85d775a9d5cdb2 \ github.com/caddy-dns/cloudflare@89f16b99c18ef49c8bb470a82f895bce01cbaece \ - github.com/caddy-dns/route53@5e0037b52d9b6dbe8ef47d5d3fdf42d7f87ebf79 \ + github.com/caddy-dns/route53@d92230e22b716e9b0c8bc1086477415f8b90e77f \ github.com/caddy-dns/duckdns@77870e12bac552ceb76917d82ced6db84b958c1f \ github.com/caddy-dns/digitalocean@9c71e343246b954976c9294a7062823605de9b9f \ github.com/caddy-dns/googleclouddns@22c91a4de6d3c3a17d395e510e1b77eab82cdc3c \ github.com/caddy-dns/gandi@d814cce86812e1e78544496e8f79e725058d8f1a \ github.com/caddy-dns/azure@f2351591d9f258201499abc37d054b7e6366fefb \ github.com/caddy-dns/porkbun@70de9b4c18f94dd2203927ab00ba104d62cb99a8 \ - github.com/caddy-dns/ovh@f71a5c6fd0073f94dd24e49233775d9b087dfe5d \ + github.com/caddy-dns/ovh@62cc061d0f87156769feb16b6a81e97462ef6cee \ github.com/caddy-dns/namecheap@7095083a353829fc83632c34e8988fd8eb72f43d \ github.com/caddy-dns/netlify@eaa9514e3b9fda329b317b937e2c6c0f23d11356 \ github.com/caddy-dns/acmedns@18621dd3e69e048eae80c4171ef56cb576dce2f4 \ github.com/caddy-dns/desec@822a6a2014b221e8fa589fbcfd0395abe9ee90f6 \ - github.com/caddy-dns/powerdns@79c99dcd21421184998486265ad3242f79b8bda6 \ + github.com/caddy-dns/powerdns@fbd76808d64f57c80d4d62587dd14b14f06aefc7 \ github.com/caddy-dns/ddnss@7f65108b0a6249d8e630fe2431143069c4317ee4 \ github.com/caddy-dns/njalla@57869f89026a2e8980d1b3fac5687e115e9acb36 \ github.com/caddy-dns/linode@6fa218b5e8d6495dd96359b5550937f10234b360 \ @@ -119,7 +119,7 @@ CADDY_CUSTOM_PLUGINS= github.com/caddyserver/ntlm-transport@e0c1e46a30093fa243d github.com/caddy-dns/dinahosting@38b1acca4e37dac795cdd2ec239acb4fc3df7fef \ github.com/caddy-dns/ionos@041b720e83ffd1245086edf4b0259a802fc586ba \ github.com/caddy-dns/hexonet@2df0595f17b1cae63394c9488eec55f4c1b63650 \ - github.com/caddy-dns/mailinabox@46af20439f1f0b8e7fdd65c2069b77d3c2c96ef1 \ + github.com/caddy-dns/mailinabox@39d0e3ce8e259f6d1b98b6c417fc79a0a1708e91 \ github.com/caddy-dns/netcup@a811da94403509715bd149669b07544706fd6d46 \ github.com/caddy-dns/rfc2136@b8df5e8730c9dcd6fce4b483530b96dcd46c0690 \ github.com/caddy-dns/dnsmadeeasy@91d629f293a577f1be3bb57529589ce39f4935b5 \ From f711ef445a173e90cd59e2c6343060dcdf395ac7 Mon Sep 17 00:00:00 2001 From: Franco Fichtner Date: Mon, 23 Sep 2024 09:24:01 +0200 Subject: [PATCH 6/9] config: need newer bind --- config/24.7/ports.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/24.7/ports.conf b/config/24.7/ports.conf index fa4b8306..1f57d030 100644 --- a/config/24.7/ports.conf +++ b/config/24.7/ports.conf @@ -47,6 +47,7 @@ devel/py-ujson@py${PRODUCT_PYTHON} devel/scons dns/bind-tools dns/bind918 arm +dns/bind920 arm dns/ddclient dns/dnscrypt-proxy2 arm dns/dnsmasq From 491718a9b3a1e55c4638cf387f760592fed5a7df Mon Sep 17 00:00:00 2001 From: Franco Fichtner Date: Mon, 23 Sep 2024 09:27:07 +0200 Subject: [PATCH 7/9] README: restructure slightly --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6df4461e..0be52a00 100644 --- a/README.md +++ b/README.md @@ -386,6 +386,15 @@ Available options are: * unused: copy unused upstream changes * (none): all of the above +Syncing a ports branch for custom package builds +------------------------------------------------ + +When maintaining branches the master branch holds updates that +we want to cherry-pick to another branch. To ease the process +the sync step can deal with the complexity involved: + + # make sync-category/port[,category/port[,...]] + Rebasing the file lists for the base sets ----------------------------------------- @@ -422,15 +431,6 @@ for standalone use on the host use: # make make.conf -Syncing a ports branch ----------------------- - -When maintaining branches the master branch holds updates that -we want to cherry-pick to another branch. To ease the process -the sync step can deal with the complexity involved: - - # make sync-category/port[,category/port[,...]] - Reading and modifying version numbers of build sets and images -------------------------------------------------------------- From 9e7dfd4c3f1a932081fdc0fb12d33f7a1de38681 Mon Sep 17 00:00:00 2001 From: Franco Fichtner Date: Tue, 24 Sep 2024 10:36:15 +0200 Subject: [PATCH 8/9] config: add mod_proxy_msprc package and pin APACHE version --- Makefile | 2 +- build/make.conf.sh | 2 +- config/24.7/build.conf | 1 + config/24.7/make.conf | 4 +++- config/24.7/ports.conf | 3 ++- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 52ac4d47..ddca322f 100644 --- a/Makefile +++ b/Makefile @@ -165,7 +165,7 @@ VERBOSE_FLAGS= -x VERBOSE_HIDDEN= @ .endif -.for _VERSION in ABI DEBUG LUA PERL PHP PYTHON RUBY SSL VERSION ZFS +.for _VERSION in ABI APACHE DEBUG LUA PERL PHP PYTHON RUBY SSL VERSION ZFS VERSIONS+= PRODUCT_${_VERSION}=${${_VERSION}} .endfor diff --git a/build/make.conf.sh b/build/make.conf.sh index 4d960123..758feffd 100644 --- a/build/make.conf.sh +++ b/build/make.conf.sh @@ -33,7 +33,7 @@ SELF=make.conf SED=sed -for KEY in LUA PERL PHP PYTHON RUBY SSL; do +for KEY in APACHE LUA PERL PHP PYTHON RUBY SSL; do eval VALUE="\${PRODUCT_${KEY}}" SED="${SED} -e s:%%${KEY}%%:${VALUE}:g" done diff --git a/config/24.7/build.conf b/config/24.7/build.conf index 8d0bbb59..91e23ae9 100644 --- a/config/24.7/build.conf +++ b/config/24.7/build.conf @@ -1,3 +1,4 @@ +APACHE?= 24 LUA?= 5.4 OS?= 14.1 PERL?= 5.36 diff --git a/config/24.7/make.conf b/config/24.7/make.conf index 8eff9fde..3468b12d 100644 --- a/config/24.7/make.conf +++ b/config/24.7/make.conf @@ -11,6 +11,7 @@ PRODUCT_PHP?= ${_PRODUCT_PHP:[2]:S/./ /g:[1..2]:tW:S/ //} # fallbacks for standard builds using opnsense-code +PRODUCT_APACHE?= %%APACHE%% PRODUCT_LUA?= %%LUA%% PRODUCT_PERL?= %%PERL%% PRODUCT_PHP?= %%PHP%% @@ -25,7 +26,8 @@ PRODUCT_GSSAPI?= GSSAPI_MIT OPTIONS_SET= MONPLUGINS OPTIONS_UNSET= DBUS DOCS EXAMPLES GCC GSSAPI_BASE NAGPLUGINS \ NLS OPENGL WAYLAND X11 -DEFAULT_VERSIONS= lua=${PRODUCT_LUA} +DEFAULT_VERSIONS= apache=${PRODUCT_APACHE:C/^./&./} +DEFAULT_VERSIONS+= lua=${PRODUCT_LUA} DEFAULT_VERSIONS+= perl5=${PRODUCT_PERL} DEFAULT_VERSIONS+= php=${PRODUCT_PHP} DEFAULT_VERSIONS+= python3=${PRODUCT_PYTHON:C/^./&./} diff --git a/config/24.7/ports.conf b/config/24.7/ports.conf index 1f57d030..fa78d5c2 100644 --- a/config/24.7/ports.conf +++ b/config/24.7/ports.conf @@ -159,6 +159,7 @@ opnsense/google-api-php-client@php${PRODUCT_PHP} opnsense/ifinfo opnsense/installer opnsense/lang +opnsense/mod_proxy_msrpc arm opnsense/netmap-bridge arm opnsense/pam opnsense/phpseclib@php${PRODUCT_PHP} @@ -240,7 +241,7 @@ textproc/php${PRODUCT_PHP}-ctype textproc/php${PRODUCT_PHP}-dom textproc/php${PRODUCT_PHP}-simplexml textproc/php${PRODUCT_PHP}-xml -www/apache24 arm +www/apache${PRODUCT_APACHE} arm www/c-icap arm www/c-icap-modules arm www/caddy-custom arm From f0def57de4a17fcddb3cd0eff560e5c1e9d256be Mon Sep 17 00:00:00 2001 From: Franco Fichtner Date: Tue, 24 Sep 2024 12:35:02 +0200 Subject: [PATCH 9/9] config: add py-ldap3 to packages --- config/24.7/ports.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/24.7/ports.conf b/config/24.7/ports.conf index fa78d5c2..a3468742 100644 --- a/config/24.7/ports.conf +++ b/config/24.7/ports.conf @@ -135,6 +135,7 @@ net/php${PRODUCT_PHP}-ldap net/php${PRODUCT_PHP}-soap arm net/php${PRODUCT_PHP}-sockets net/pimd arm +net/py-ldap3@py${PRODUCT_PYTHON} net/py-netaddr@py${PRODUCT_PYTHON} net/py-speedtest-cli@py${PRODUCT_PYTHON} net/radsecproxy arm