Skip to content

Commit

Permalink
MT#61993 build: Rework to use a config.mk generated during the build
Browse files Browse the repository at this point in the history
These are potentially computed from inside each subdir, and in addition
due to what appears to be a regression in GNU make 4.4, where it is
reevaluating variables that contain $(shell) functions, many times (in
the order of thousands, this was slowing down the build, were on the
Debian amd64 build daemons it went from 5m with GNU make 4.3 to 2h40m
with GNU make 4.4. Although the bulk of the slow down has been fixed
with previous commits, the remaining optimizations are only to avoid
this potentially happening again in the future, and to reduce useless
duplicate work.

Instead of trying to cache the values from within make itself, where
programming this there is extremely painful, and does not seem to be
able to greatly reduce the number of calls, because the build system
is going to be called multiple times for different targets. Simply
externalize the generation into several shell scripts, that we call
to generate a make fragment that then we include from the various
Makefiles.

For a Debian build with GNU make 4.3, this reduces the amount of total
pkg-config calls from around ~1600 to 128, for dpkg-buildflags from
~1100 down to 6, and for dpkg-parsechangelog from ~56 to 17, but the
slow down is not as significant there anyway.

For a Debian build with GNU make 4.4, this reduces the amount of total
pkg-config calls from around ~2600 to 128, for dpkg-buildflags from
~2800 down to 6, and for dpkg-parsechangelog from ~350 to 21.

For a Debian build with GNU make 4.4, this reduces the build time
on this system from 2m10s to ~ 1m30s.

Change-Id: I427d0ea5106dc6ed1ff9e664ccdba2fa0725b7d0
  • Loading branch information
guillemj committed Jan 28, 2025
1 parent bb2bf3a commit 7dbe24e
Show file tree
Hide file tree
Showing 18 changed files with 416 additions and 309 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
*.strhash.c
docs/_*
bencode.c
config.mk
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
.DEFAULT_GOAL := all

RTPENGINE_ROOT_DIR=.
with_transcoding ?= yes

ifeq ($(DO_ASAN_FLAGS),1)
Expand All @@ -20,6 +19,11 @@ export UBSAN_OPTIONS=print_stacktrace=1
export G_SLICE=always-malloc
endif

export top_srcdir = $(CURDIR)

# Initialize all flags, so that we only compute them once.
include lib/deps.Makefile

include lib/lib.Makefile

.PHONY: all distclean clean coverity
Expand Down Expand Up @@ -63,6 +67,7 @@ distclean clean:
$(MAKE) -C perf-tester clean
$(MAKE) -C kernel-module clean
$(MAKE) -C t clean
rm -f config.mk

.DEFAULT:
$(MAKE) -C daemon $@
Expand Down
109 changes: 56 additions & 53 deletions daemon/Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
TARGET= rtpengine

export top_srcdir = ..

include ../lib/deps.Makefile

with_iptables_option ?= yes
with_transcoding ?= yes

Expand All @@ -11,36 +15,33 @@ CFLAGS:= $(CFLAGS)
endif
CFLAGS+= -pthread
CFLAGS+= -std=c11
CFLAGS+= $(shell pkg-config --cflags glib-2.0)
CFLAGS+= $(shell pkg-config --cflags gthread-2.0)
CFLAGS+= $(shell pkg-config --cflags zlib)
CFLAGS+= $(shell pkg-config --cflags openssl)
CFLAGS+= $(shell pkg-config --cflags libevent_pthreads)
CFLAGS+= -DPCRE2_CODE_UNIT_WIDTH=8
CFLAGS+= $(shell pkg-config --cflags libpcre2-8)
CFLAGS+= $(shell pkg-config xmlrpc_client --cflags 2> /dev/null || xmlrpc-c-config client --cflags)
CFLAGS+= $(shell pkg-config xmlrpc --cflags 2> /dev/null)
CFLAGS+= $(shell pkg-config xmlrpc_util --cflags 2> /dev/null)
CFLAGS+= $(shell pkg-config --cflags json-glib-1.0)
CFLAGS+= $(shell pkg-config --cflags libwebsockets)
CFLAGS+= $(shell pkg-config --cflags libnftnl)
CFLAGS+= $(shell pkg-config --cflags libmnl)
CFLAGS+= -I. -I../kernel-module/ -I../lib/ -I../include/
CFLAGS+= -D_GNU_SOURCE
CFLAGS+= $(CFLAGS_GLIB)
CFLAGS+= $(CFLAGS_GTHREAD)
CFLAGS+= $(CFLAGS_LIBZ)
CFLAGS+= $(CFLAGS_OPENSSL)
CFLAGS+= $(CFLAGS_LIBEVENT)
CFLAGS+= $(CFLAGS_LIBPCRE)
CFLAGS+= $(CFLAGS_XMLRPC)
CFLAGS+= $(CFLAGS_JSON_GLIB)
CFLAGS+= $(CFLAGS_LIBWEBSOCKETS)
CFLAGS+= $(CFLAGS_LIBNFTNL)
CFLAGS+= $(CFLAGS_LIBMNL)
ifeq ($(with_iptables_option),yes)
CFLAGS+= $(shell pkg-config --cflags libiptc)
CFLAGS+= $(CFLAGS_LIBIPTC)
CFLAGS+= -DWITH_IPTABLES_OPTION
endif
CFLAGS+= -I. -I../kernel-module/ -I../lib/ -I../include/
CFLAGS+= -D_GNU_SOURCE
ifeq ($(with_transcoding),yes)
CFLAGS+= $(shell pkg-config --cflags libavcodec)
CFLAGS+= $(shell pkg-config --cflags libavformat)
CFLAGS+= $(shell pkg-config --cflags libavutil)
CFLAGS+= $(shell pkg-config --cflags libswresample)
CFLAGS+= $(shell pkg-config --cflags libavfilter)
CFLAGS+= $(shell pkg-config --cflags spandsp)
CFLAGS+= $(shell pkg-config --cflags opus)
CFLAGS+= $(CFLAGS_LIBAVCODEC)
CFLAGS+= $(CFLAGS_LIBAVFORMAT)
CFLAGS+= $(CFLAGS_LIBAVUTIL)
CFLAGS+= $(CFLAGS_LIBSWRESAMPLE)
CFLAGS+= $(CFLAGS_LIBAVFILTER)
CFLAGS+= $(CFLAGS_SPANDSP)
CFLAGS+= $(CFLAGS_OPUS)
CFLAGS+= -DWITH_TRANSCODING
CFLAGS+= $(shell mysql_config --cflags)
CFLAGS+= $(CFLAGS_MYSQL)
else
CFLAGS+= -DWITHOUT_CODECLIB
endif
Expand All @@ -53,42 +54,44 @@ endif
#CFLAGS+= -DSTRICT_SDES_KEY_LIFETIME

LDLIBS:= -lm -ldl
LDLIBS+= $(shell pkg-config --libs glib-2.0)
LDLIBS+= $(shell pkg-config --libs gthread-2.0)
LDLIBS+= $(shell pkg-config --libs zlib)
LDLIBS+= $(shell pkg-config --libs libpcre2-8)
LDLIBS+= $(shell pkg-config --libs libcrypto)
LDLIBS+= $(shell pkg-config --libs openssl)
LDLIBS+= $(shell pkg-config --libs libevent_pthreads)
LDLIBS+= -lpcap
LDLIBS+= $(shell pkg-config xmlrpc_client --libs 2> /dev/null || xmlrpc-c-config client --libs)
LDLIBS+= $(shell pkg-config xmlrpc --libs 2> /dev/null)
LDLIBS+= $(shell pkg-config xmlrpc_util --libs 2> /dev/null)
LDLIBS+= -lhiredis
LDLIBS+= $(shell pkg-config --libs json-glib-1.0)
LDLIBS+= $(shell pkg-config --libs libwebsockets)
LDLIBS+= $(shell pkg-config --libs libnftnl)
LDLIBS+= $(shell pkg-config --libs libmnl)
LDLIBS+= $(LDLIBS_GLIB)
LDLIBS+= $(LDLIBS_GTHREAD)
LDLIBS+= $(LDLIBS_LIBZ)
LDLIBS+= $(LDLIBS_LIBPCRE)
LDLIBS+= $(LDLIBS_LIBCRYPTO)
LDLIBS+= $(LDLIBS_OPENSSL)
LDLIBS+= $(LDLIBS_LIBEVENT)
LDLIBS+= $(LDLIBS_LIBPCAP)
LDLIBS+= $(LDLIBS_XMLRPC)
LDLIBS+= $(LDLIBS_LIBHIREDIS)
LDLIBS+= $(LDLIBS_JSON_GLIB)
LDLIBS+= $(LDLIBS_LIBWEBSOCKETS)
LDLIBS+= $(LDLIBS_LIBNFTNL)
LDLIBS+= $(LDLIBS_LIBMNL)
ifeq ($(with_iptables_option),yes)
LDLIBS+= $(shell pkg-config --libs libiptc)
LDLIBS+= $(LDLIBS_LIBIPTC)
endif
ifeq ($(with_transcoding),yes)
LDLIBS+= $(shell pkg-config --libs libavcodec)
LDLIBS+= $(shell pkg-config --libs libavformat)
LDLIBS+= $(shell pkg-config --libs libavutil)
LDLIBS+= $(shell pkg-config --libs libswresample)
LDLIBS+= $(shell pkg-config --libs libavfilter)
LDLIBS+= $(shell pkg-config --libs spandsp)
LDLIBS+= $(shell pkg-config --libs opus)
LDLIBS+= $(shell mysql_config --libs)
LDLIBS+= $(LDLIBS_LIBAVCODEC)
LDLIBS+= $(LDLIBS_LIBAVFORMAT)
LDLIBS+= $(LDLIBS_LIBAVUTIL)
LDLIBS+= $(LDLIBS_LIBSWRESAMPLE)
LDLIBS+= $(LDLIBS_LIBAVFILTER)
LDLIBS+= $(LDLIBS_SPANDSP)
LDLIBS+= $(LDLIBS_OPUS)
LDLIBS+= $(LDLIBS_MYSQL)
endif

ifeq ($(with_transcoding),yes)
include ../lib/g729.Makefile
include ../lib/codec-chain.Makefile
CFLAGS+= $(CFLAGS_BCG729)
LDLIBS+= $(LDLIBS_BCG729)

CFLAGS+= $(CFLAGS_CODEC_CHAIN)
LDLIBS+= $(LDLIBS_CODEC_CHAIN)
endif

include ../lib/mqtt.Makefile
CFLAGS+= $(CFLAGS_MQTT)
LDLIBS+= $(LDLIBS_MQTT)

SRCS= main.c kernel.c helpers.c control_tcp.c call.c control_udp.c redis.c \
cookie_cache.c udp_listener.c control_ng_flags_parser.c control_ng.strhash.c sdp.strhash.c stun.c rtcp.c \
Expand Down
1 change: 1 addition & 0 deletions debian/ngcp-rtpengine-kernel-dkms.install
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ kernel-module/*.c usr/src/${env:DEB_SOURCE}-${env:DEB_VERSION_UPSTREAM}
kernel-module/*.h usr/src/${env:DEB_SOURCE}-${env:DEB_VERSION_UPSTREAM}
kernel-module/*.inc usr/src/${env:DEB_SOURCE}-${env:DEB_VERSION_UPSTREAM}
kernel-module/Makefile usr/src/${env:DEB_SOURCE}-${env:DEB_VERSION_UPSTREAM}
kernel-module/gen-* usr/src/${env:DEB_SOURCE}-${env:DEB_VERSION_UPSTREAM}
1 change: 1 addition & 0 deletions kernel-module/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@
*.mod.c
modules.order
Module.symvers
rtpengine.mk
.*.cmd
xt_RTPENGINE.mod
38 changes: 13 additions & 25 deletions kernel-module/Makefile
Original file line number Diff line number Diff line change
@@ -1,42 +1,30 @@
PWD := $(shell pwd)
KSRC ?= /lib/modules/$(shell uname -r)/build
KBUILD := $(KSRC)
M ?= $(PWD)

ifeq ($(RTPENGINE_VERSION),)
DPKG_PRSCHNGLG := $(shell which dpkg-parsechangelog 2>/dev/null)
DEB_CHANGELOG := $(shell test -f $(M)/../debian/changelog && echo $(M)/../debian/changelog || echo $(M)/debian/changelog)
ifneq ($(DPKG_PRSCHNGLG),)
DPKG_PRSCHNGLG := $(shell dpkg-parsechangelog -l$(DEB_CHANGELOG) | awk '/^Version: / {print $$2}')
endif
GIT_BR_COMMIT := git-$(shell cd $(M) && git rev-parse --abbrev-ref --symbolic-full-name HEAD 2> /dev/null)-$(shell cd $(M) && git rev-parse --short HEAD 2> /dev/null)

ifneq ($(DPKG_PRSCHNGLG),)
RTPENGINE_VERSION+=$(DPKG_PRSCHNGLG)
endif
ifneq ($(GIT_BR_COMMIT),git--)
RTPENGINE_VERSION+=$(GIT_BR_COMMIT)
endif

ifeq ($(RTPENGINE_VERSION),)
RTPENGINE_VERSION+=undefined
endif
endif
M ?= $(CURDIR)

export M

include rtpengine-kmod.mk

EXTRA_CFLAGS+= -DRTPENGINE_VERSION="\"$(RTPENGINE_VERSION)\""

obj-m += xt_RTPENGINE.o

.PHONY: modules clean patch install

modules:
$(MAKE) -C $(KBUILD) M=$(PWD) O=$(KBUILD) modules
$(MAKE) -C $(KBUILD) M=$(CURDIR) O=$(KBUILD) modules

clean:
$(MAKE) -C $(KBUILD) M=$(PWD) clean || true
$(MAKE) -C $(KBUILD) M=$(CURDIR) clean || true
rm -f rtpengine-kmod.mk

patch:
../utils/patch-kernel magic "$(PWD)" "$(KERNEL)" "$(RTPENGINE_VERSION)"
../utils/patch-kernel magic "$(CURDIR)" "$(KERNEL)" "$(RTPENGINE_VERSION)"

install:
install -D xt_RTPENGINE.ko $(DESTDIR)/lib/modules/$(shell uname -r)/updates/xt_RTPENGINE.ko
depmod -a

rtpengine-kmod.mk:
./gen-rtpengine-kmod-flags >$@
33 changes: 33 additions & 0 deletions kernel-module/gen-rtpengine-kmod-flags
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash

if [ -z "${M}" ]; then
M=$(pwd)
fi

if [ -z "${RTPENGINE_VERSION}" ]; then
have_dpkg_parsechangelog=no
if command -v dpkg-parsechangelog >/dev/null; then
have_dpkg_parsechangelog=yes
fi
if [ -f "${M}/../debian/changelog" ]; then
deb_changelog="${M}/../debian/changelog"
else
deb_changelog="${M}/debian/changelog"
fi
if [ "${have_dpkg_parsechangelog}" = yes ]; then
deb_version="$(dpkg-parsechangelog -l${deb_changelog} | awk '/^Version: / {print $$2}')"
fi
git_br_commit="git-$(cd ${M} && git rev-parse --abbrev-ref --symbolic-full-name HEAD 2> /dev/null)-$(cd ${M} && git rev-parse --short HEAD 2> /dev/null)"

if [ "${have_dpkg_parsechangelog}" = yes ]; then
RTPENGINE_VERSION+=" ${deb_version}"
fi
if [ "${git_br_commit}" != "git--" ]; then
RTPENGINE_VERSION+=" ${git_br_commit}"
fi

if [ -z "${RTPENGINE_VERSION}" ]; then
RTPENGINE_VERSION="undefined"
fi
echo "RTPENGINE_VERSION := ${RTPENGINE_VERSION}"
fi
7 changes: 0 additions & 7 deletions lib/codec-chain.Makefile

This file was deleted.

12 changes: 12 additions & 0 deletions lib/deps.Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Define build flags for used dependencies.

$(top_srcdir)/config.mk:
$(top_srcdir)/utils/gen-common-flags >$@
ifeq (,$(filter pkg.ngcp-rtpengine.nobcg729,${DEB_BUILD_PROFILES}))
$(top_srcdir)/utils/gen-bcg729-flags >>$@
endif
ifneq (,$(filter pkg.ngcp-rtpengine.codec-chain,${DEB_BUILD_PROFILES}))
$(top_srcdir)/utils/gen-codec-chain-flags >>$@
endif

include $(top_srcdir)/config.mk
58 changes: 0 additions & 58 deletions lib/g729.Makefile

This file was deleted.

Loading

0 comments on commit 7dbe24e

Please sign in to comment.