diff --git a/db/schema/opensips-trie.xml b/db/schema/opensips-trie.xml new file mode 100644 index 00000000000..fc55d355500 --- /dev/null +++ b/db/schema/opensips-trie.xml @@ -0,0 +1,14 @@ + + +%entities; + +]> + + + Trie + + + diff --git a/db/schema/trie_partitions.xml b/db/schema/trie_partitions.xml new file mode 100644 index 00000000000..14b317c5fb5 --- /dev/null +++ b/db/schema/trie_partitions.xml @@ -0,0 +1,56 @@ + + +%entities; + +]> + + + dr_partitions + 1 + &MYSQL_TABLE_TYPE; + + This table is used by the Trie module to store + information about the partitions used in the script (url to database and table name). + More information can be found at: &OPENSIPS_MOD_DOC;trie.html. + + + + + id + unsigned int + &table_id_len; + + + + int,auto + Partition unique ID + + + + + partition_name + string + 255 + The name of the partition. + + + + + db_url + string + 255 + The url to the database containing the tables: dr_rules, dr_groups, + dr_carriers and dr_gateways + + + + trie_table + string + 255 + + The name of the trie_rules table in the given database (for the given partition). + +
diff --git a/db/schema/trie_table.xml b/db/schema/trie_table.xml new file mode 100644 index 00000000000..da9e87b8774 --- /dev/null +++ b/db/schema/trie_table.xml @@ -0,0 +1,57 @@ + + +%entities; + +]> + + + trie_table + 1 + &MYSQL_TABLE_TYPE; + + This table is used by the Trie module in order to build the trie that it caches + More information can be found at: &OPENSIPS_MOD_DOC;trie.html. + + + + + ruleid + unsigned int + &table_id_len; + + + + int,auto + Rule unique ID + + + + + prefix + string + 64 + prefix to be cached + + + + attrs + string + 255 + + + Generic string describing RULE attributes - this string is + to be interpreted from the script + + + + priority + int + 11 + 1 + 1 if the rule is enabled, 0 if the rule is disabled. + + +
diff --git a/modules/trie/Makefile b/modules/trie/Makefile new file mode 100644 index 00000000000..24639228aaa --- /dev/null +++ b/modules/trie/Makefile @@ -0,0 +1,10 @@ +# $Id$ +# +# WARNING: do not run this directly, it should be run by the master Makefile + +include ../../Makefile.defs +auto_gen= +NAME=trie.so +LIBS= + +include ../../Makefile.modules diff --git a/modules/trie/README b/modules/trie/README new file mode 100644 index 00000000000..7f80b44c9a5 --- /dev/null +++ b/modules/trie/README @@ -0,0 +1,277 @@ +Trie Module + __________________________________________________________ + + Table of Contents + + 1. Admin Guide + + 1.1. Overview + + 1.1.1. Introduction + + 1.2. Dependencies + + 1.2.1. OpenSIPS Modules + 1.2.2. External Libraries or Applications + + 1.3. Exported Parameters + + 1.3.1. trie_table(str) + 1.3.2. no_concurrent_reload (int) + 1.3.3. use_partitions (int) + 1.3.4. db_partitions_url (str) + 1.3.5. db_partitions_table (str) + 1.3.6. extra_prefix_chars (str) + + 1.4. Exported Functions + + 1.4.1. trie_search(number, [flags], + [trie_attrs_pvar], [match_prefix_pvar], + [partition]) + + 1.5. Exported MI Functions + + 1.5.1. trie_reload + 1.5.2. trie_reload_status + 1.5.3. trie_search + 1.5.4. trie_number_delete + 1.5.5. trie_number_upsert + + 1.6. Installation + + List of Examples + + 1.1. Set trie_table parameter + 1.2. Set no_concurrent_reload parameter + 1.3. Set use_partitions parameter + 1.4. Set db_partitions_url parameter + 1.5. Set db_partitions_table parameter + 1.6. Set extra_prefix_chars parameter + 1.7. trie_search usage + 1.8. trie_reload_status usage when use_partitions is 0 + +Chapter 1. Admin Guide + +1.1. Overview + +1.1.1. Introduction + + Trie is a module for efficiently caching and lookup of a set of + prefixes ( stored in a trie data structure ) + +1.2. Dependencies + +1.2.1. OpenSIPS Modules + + The following modules must be loaded before this module: + * a database module. + +1.2.2. External Libraries or Applications + + * none. + +1.3. Exported Parameters + +1.3.1. trie_table(str) + + The name of the db table storing prefix rules. + + Default value is “trie_table”. + + Example 1.1. Set trie_table parameter +... +modparam("drouting", "trie_table", "my_prefix_table") +... + +1.3.2. no_concurrent_reload (int) + + If enabled, the module will not allow do run multiple + trie_reload MI commands in parallel (with overlapping) Any new + reload will be rejected (and discarded) while an existing + reload is in progress. + + If you have a large routing set (millions of rules/prefixes), + you should consider disabling concurrent reload as they will + exhaust the shared memory (by reloading into memory, in the + same time, multiple instances of routing data). + + Default value is “0 (disabled)”. + + Example 1.2. Set no_concurrent_reload parameter +... +# do not allow parallel reload operations +modparam("trie", "no_concurrent_reload", 1) +... + +1.3.3. use_partitions (int) + + Flag to configure whether to use partitions for tries. If this + flag is set then the db_partitions_url and db_partitions_table + variables become mandatory. + + Default value is “0”. + + Example 1.3. Set use_partitions parameter +... +modparam("trie", "use_partitions", 1) +... + +1.3.4. db_partitions_url (str) + + The url to the database containing partition-specific + information.The use_partitions parameter must be set to 1. + + Default value is “"NULL"”. + + Example 1.4. Set db_partitions_url parameter +... +modparam("trie", "db_partitions_url", "mysql://user:password@localhost/o +pensips_partitions") +... + +1.3.5. db_partitions_table (str) + + The name of the table containing partition definitions. To be + used with use_partitions and db_partitions_url. + + Default value is “trie_partitions”. + + Example 1.5. Set db_partitions_table parameter +... +modparam("trie", "db_partitions_table", "trie_partition_defs") +... + +1.3.6. extra_prefix_chars (str) + + List of ASCII (0-127) characters to be additionally accepted in + the prefixes. By default only '0' - '9' chars (digits) are + accepted. + + Default value is “NULL”. + + Example 1.6. Set extra_prefix_chars parameter +... +modparam("trie", "extra_prefix_chars", "#-%") +... + +1.4. Exported Functions + +1.4.1. trie_search(number, [flags], [trie_attrs_pvar], +[match_prefix_pvar], [partition]) + + Function to search for an entry ( number ) in a trie. + + This function can be used from all routes. + + If you set use_partitions to 1 the partition last parameter + becomes mandatory. + + All parameters are optional. Any of them may be ignored, + provided the necessary separation marks "," are properly + placed. + * number (str) - number to be searched in the trie + * flags (string, optional) - a list of letter-like flags for + controlling the routing behavior. Possible flags are: + + L - Do strict length matching over the prefix - + actually the trie engine will do full number matching + and not prefix matching anymore. + * trie_attrs_pvar (var, optional) - a writable variable which + will be populated with the attributes of the matched trie + rule. + * match_prefix_pvar (var, optional) - a writable variable + which will be the actual prefix matched in the trie. + * partition (string, optional) - the name of the trie + partition to be used. This parameter is to be defined ONLY + if the "use_partition" module parameter is turned on. + + Example 1.7. trie_search usage +... +if (trie_search("$rU","L",$avp(code_attrs),,"my_partition")) { + # we found it in the trie, it's a match + xlog("We found $rU in the trie with attrs $avp(code_attrs) \n"); +} + +1.5. Exported MI Functions + +1.5.1. trie_reload + + Command to reload trie rules from database. + * if use_partition is set to 0 - all routing rules will be + reloaded. + * if use_partition is set to 1, the parameters are: + + partition_name (optional) - if not provided all the + partitions will be reloaded, otherwise just the + partition given as parameter will be reloaded. + + MI FIFO Command Format: + opensips-cli -x mi trie_reload part_1 + +1.5.2. trie_reload_status + + Gets the time of the last reload for any partition. + * if use_partition is set to 0 - the function doesn't receive + any parameter. It will list the date of the last reload for + the default (and only) partition. + * if use_partition is set to 1, the parameters are: + + partition_name (optional) - if not provided the + function will list the time of the last update for + every partition. Otherwise, the function will list the + time of the last reload for the given partition. + + Example 1.8. trie_reload_status usage when use_partitions is 0 +$ opensips-cli -x mi dr_reload_status +Date:: Tue Aug 12 12:26:00 2014 + +1.5.3. trie_search + + Tries to match a number in the existing tries loaded from the + database. + * if use_partition is set to 1 the function will have 2 + parameters: + + partition_name + + number - the number to test against + * if use_partition is set to 0 the function will have 1 + parameter: + + number - the number to test against + + MI FIFO Command Format: + opensips-cli -x mi trie_search partition_name=part1 numb +er=012340987 + +1.5.4. trie_number_delete + + Deletes individual entries in the trie, without reloading all + of the data + * if use_partition is set to 1 the function will have 2 + parameters: + + partition_name + + number - the array of numbers to delete + + MI FIFO Command Format: + opensips-cli -x mi trie_number_delete partition_name=par +t1 number=["012340987","4858345"] + +1.5.5. trie_number_upsert + + Upserts ( insert if not found, update is found ) an array of + numbers in the trie, without reloading all of the data + * if use_partition is set to 1 the function will have 3 + parameters: + + partition_name + + number - the array of numbers to update + + attrs - the array of new attributes for the numbers + + MI FIFO Command Format: + opensips-cli -x mi trie_number_upsert partition_name=par +t1 number=["012340987"] attrs=["my_attrs"] + +1.6. Installation + + The module requires some tables in the OpenSIPS database. You + can also find the complete database documentation on the + project webpage, + https://opensips.org/docs/db/db-schema-devel.html. + + Documentation Copyrights: + + Copyright © 2024 OpenSIPS Project diff --git a/modules/trie/doc/trie.xml b/modules/trie/doc/trie.xml new file mode 100644 index 00000000000..48fd719a093 --- /dev/null +++ b/modules/trie/doc/trie.xml @@ -0,0 +1,27 @@ + + + + + +%docentities; + +]> + + + + Trie Module + &osipsname; + + + + &admin; + + &docCopyrights; + ©right; 2024 OpenSIPS Project + + + diff --git a/modules/trie/doc/trie_admin.xml b/modules/trie/doc/trie_admin.xml new file mode 100644 index 00000000000..a383a447819 --- /dev/null +++ b/modules/trie/doc/trie_admin.xml @@ -0,0 +1,453 @@ + + + + &adminguide; + +
+ Overview +
+ Introduction + + Trie is a module for efficiently caching and lookup of a set of prefixes ( stored in a trie data structure ) + +
+ +
+ + +
+ Dependencies +
+ &osips; Modules + + The following modules must be loaded before this module: + + + + + a database module. + + + + +
+ +
+ External Libraries or Applications + + + + none. + + + + +
+
+ +
+ Exported Parameters +
+ <varname>trie_table</varname>(str) + + The name of the db table storing prefix rules. + + + Default value is trie_table. + + + + Set <varname>trie_table</varname> parameter + +... +modparam("drouting", "trie_table", "my_prefix_table") +... + + +
+ +
+ <varname>no_concurrent_reload</varname> (int) + + If enabled, the module will not allow do run multiple trie_reload + MI commands in parallel (with overlapping) Any new reload will + be rejected (and discarded) while an existing reload is in + progress. + + + If you have a large routing set (millions of rules/prefixes), you + should consider disabling concurrent reload as they will exhaust + the shared memory (by reloading into memory, in the same time, + multiple instances of routing data). + + + Default value is 0 (disabled). + + + + Set <varname>no_concurrent_reload</varname> parameter + +... +# do not allow parallel reload operations +modparam("trie", "no_concurrent_reload", 1) +... + + +
+ +
+ <varname>use_partitions</varname> (int) + + Flag to configure whether to use partitions for tries. If this + flag is set then the db_partitions_url and + db_partitions_table + variables become mandatory. + + + Default value is 0. + + + + Set <varname>use_partitions</varname> parameter + +... +modparam("trie", "use_partitions", 1) +... + + +
+ +
+ <varname>db_partitions_url</varname> (str) + + The url to the database containing partition-specific + information.The use_partitions parameter + must be set to 1. + + + Default value is "NULL". + + + + Set <varname>db_partitions_url</varname> parameter + +... +modparam("trie", "db_partitions_url", "mysql://user:password@localhost/opensips_partitions") +... + + +
+ +
+ <varname>db_partitions_table</varname> (str) + + The name of the table containing partition definitions. To be + used with use_partitions and db_partitions_url. + + + Default value is trie_partitions. + + + + Set <varname>db_partitions_table</varname> parameter + +... +modparam("trie", "db_partitions_table", "trie_partition_defs") +... + + +
+ +
+ <varname>extra_prefix_chars</varname> (str) + + List of ASCII (0-127) characters to be additionally accepted in + the prefixes. By default only '0' - '9' chars (digits) are + accepted. + + + Default value is NULL. + + + + Set <varname>extra_prefix_chars</varname> parameter + +... +modparam("trie", "extra_prefix_chars", "#-%") +... + + +
+
+ +
+ Exported Functions + +
+ + +
+ Exported MI Functions +
+ + <function moreinfo="none">trie_reload</function> + + + Command to reload trie rules from database. + + + + + if use_partition is set to 0 - all routing rules will be reloaded. + + + + + + if use_partition is set to 1, the parameters are: + + + partition_name (optional) - if not provided + all the partitions will be reloaded, otherwise just the partition given as parameter will be reloaded. + + + + + + + + MI FIFO Command Format: + + + opensips-cli -x mi trie_reload part_1 + +
+ +
+ <varname>trie_reload_status</varname> + + Gets the time of the last reload for any partition. + + + + + if use_partition is set to 0 - the function + doesn't receive any parameter. It will list the date of the + last reload for the default (and only) partition. + + + + + if use_partition is set to 1, the parameters are: + + + partition_name (optional) - if not provided + the function will list the time of the last update for every + partition. Otherwise, the function will list the time of the last + reload for the given partition. + + + + + + + <function>trie_reload_status</function> usage when <varname>use_partitions</varname> is 0 + +$ opensips-cli -x mi dr_reload_status +Date:: Tue Aug 12 12:26:00 2014 + + +
+ +
+ <varname>trie_search</varname> + + Tries to match a number in the existing tries loaded from the database. + + + + + if use_partition is set to 1 the function + will have 2 parameters: + + + partition_name + + + number - the number to test against + + + + + + + if use_partition is set to 0 the function will have 1 parameter: + + + number - the number to test against + + + + + + + MI FIFO Command Format: + + + opensips-cli -x mi trie_search partition_name=part1 number=012340987 + +
+ +
+ + <function moreinfo="none">trie_number_delete</function> + + + Deletes individual entries in the trie, without reloading all of the data + + + + + + if use_partition is set to 1 the function + will have 2 parameters: + + + partition_name + + + number - the array of numbers to delete + + + + + + + MI FIFO Command Format: + + + opensips-cli -x mi trie_number_delete partition_name=part1 number=["012340987","4858345"] + +
+ +
+ + <function moreinfo="none">trie_number_upsert</function> + + + Upserts ( insert if not found, update is found ) an array of numbers in the trie, without reloading all of the data + + + + + + if use_partition is set to 1 the function + will have 3 parameters: + + + partition_name + + + number - the array of numbers to update + + + attrs - the array of new attributes for the numbers + + + + + + + MI FIFO Command Format: + + + opensips-cli -x mi trie_number_upsert partition_name=part1 number=["012340987"] attrs=["my_attrs"] + +
+ +
+ + +
+ Installation + + The module requires some tables in the OpenSIPS database. + You can also find the complete database documentation on the project webpage, &osipsdbdocslink;. + +
+ + diff --git a/modules/trie/prefix_tree.c b/modules/trie/prefix_tree.c new file mode 100644 index 00000000000..9dd13c43164 --- /dev/null +++ b/modules/trie/prefix_tree.c @@ -0,0 +1,316 @@ + /* + * Trie Module + * + * Copyright (C) 2024 OpenSIPS Project + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * History: + * -------- + * 2024-12-03 initial release (vlad) + */ + +#include +#include + +#include "../../str.h" +#include "../../mem/shm_mem.h" +#include "../../time_rec.h" + +#include "prefix_tree.h" +#include "trie_partitions.h" + +#define DR_PREFIX_ARRAY_SIZE 128 +static unsigned char *trie_char2idx = NULL; + +/* number of children under a prefix node */ +int ptree_children = 0; + +#define IDX_OF_CHAR(_c) \ + trie_char2idx[ (unsigned char)(_c) ] + +#define IS_VALID_PREFIX_CHAR(_c) \ + ((((unsigned char)(_c))=DR_PREFIX_ARRAY_SIZE) { + LM_ERR("extra prefix char <%c/%d> out of range (max=%d)," + " ignoring\n",extra_prefix_chars[i],extra_prefix_chars[i], + DR_PREFIX_ARRAY_SIZE); + continue; + } + IDX_OF_CHAR( extra_prefix_chars[i] ) = ptree_children++; + } + } + LM_INFO("counted %d possible chars under a node\n", ptree_children); + + return 0; +} + +trie_node_t *get_child(trie_node_t *parent, int child_index) { + char *first_child_ptr; + + if (child_index < 0 || child_index >= ptree_children) { + LM_ERR("Out of bounds child %d requested \n",child_index); + return NULL; + } + + /* get first child allocated right after current node */ + first_child_ptr = ((char*)parent) + sizeof(trie_node_t); + /* offset it to get to the right child_index */ + return *(trie_node_t**)(first_child_ptr + child_index * sizeof(trie_node_t*)); +} + +trie_info_t* +get_trie_prefix( + trie_node_t *ptree, + str* prefix, + unsigned int *matched_len, + int enabled_only + ) +{ + char *tmp=NULL,*last_valid_tmp; + char local=0; + int idx=0; + trie_info_t *last_valid = NULL; + trie_node_t *current = NULL; + + if(NULL == ptree) + goto err_exit; + if(NULL == prefix) + goto err_exit; + tmp = prefix->s; + if (tmp == NULL) + goto err_exit; + + current = ptree; + + /* go the tree down to the last digit in the + * prefix string or down to a leaf */ + while(tmp< (prefix->s+prefix->len)) { + local=*tmp; + idx = IDX_OF_CHAR(local); + if (!IS_VALID_PREFIX_CHAR(*tmp) || (current = get_child(current,idx)) == NULL) { + break; + } + + if (current->info && (!enabled_only || current->info->enabled)) { + /* found a valid node, store it */ + last_valid = current->info; + last_valid_tmp = tmp; + } + tmp++; + } + + if (last_valid && matched_len) { + *matched_len = last_valid_tmp + 1 - prefix->s; + } + + return last_valid; + +err_exit: + return NULL; +} + +int add_trie_info( + trie_node_t *pn, + trie_info_t* r, + osips_malloc_f malloc_f, + osips_free_f free_f + ) +{ + pn->info = r; + return 0; +} + +int add_trie_prefix(trie_node_t *ptree, str *prefix, trie_info_t *r, osips_malloc_f malloc_f, osips_free_f free_f) +{ + char* tmp=NULL; + int res = 0; + trie_node_t *child; + + if (ptree == NULL || prefix == NULL || prefix->s == NULL) { + LM_ERR("ptree or no prefix\n"); + return -1; + } + + tmp = prefix->s; + while(tmp < (prefix->s+prefix->len)) { + if(NULL == tmp) { + LM_ERR("prefix became null\n"); + goto err_exit; + } + if( !IS_VALID_PREFIX_CHAR(*tmp) ) { + /* unknown character in the prefix string */ + LM_ERR("%c is not valid char in the prefix\n", *tmp); + goto err_exit; + } + + /* process the current digit in the prefix */ + if( (child = get_child(ptree,IDX_OF_CHAR(*tmp))) == NULL) { + /* allocate new node */ + INIT_TRIE_NODE(malloc_f,child); + SET_TRIE_CHILD(ptree,IDX_OF_CHAR(*tmp),child); + } + + ptree = get_child(ptree,IDX_OF_CHAR(*tmp)); + + if( tmp == (prefix->s+prefix->len-1) ) { + /* last digit in the prefix string */ + LM_DBG("adding info %p, at: " + "%p (%d)\n", r, ptree, + IDX_OF_CHAR(*tmp)); + res = add_trie_info(ptree,r, malloc_f, free_f); + if(res < 0 ) { + LM_ERR("adding rt info doesn't work\n"); + goto err_exit; + } + res = 1; + goto ok_exit; + } + + tmp++; + } + +ok_exit: + return 0; + +err_exit: + return -1; +} + +int del_tree(trie_node_t* t, osips_free_f free_f) { + if (t == NULL) { + return 0; + } + + for (int i = 0; i < ptree_children; i++) { + trie_node_t *child = get_child(t, i); + if (child != NULL) { + if (child->info != NULL) { + free_trie_info(child->info, free_f); + } + del_tree(child, free_f); + } + } + + func_free(free_f, t); + + return 0; +} + +void +free_trie_info( + trie_info_t *rl, + osips_free_f f + ) +{ + if (NULL!=rl->attrs.s) + shm_free(rl->attrs.s); + func_free(f, rl); + return; +} + +trie_data_t* +build_trie_data(struct head_db *part) +{ + trie_data_t *rdata=NULL; + + if( NULL==(rdata=func_malloc(part->malloc, sizeof(trie_data_t)))) { + LM_ERR("no more shm mem\n"); + goto err_exit; + } + memset(rdata, 0, sizeof(trie_data_t)); + + /* empty trie with no children */ + INIT_TRIE_NODE(part->malloc, rdata->pt); + return rdata; +err_exit: + if (rdata) + func_free(part->free, rdata); + return 0; +} + +trie_info_t* +build_trie_info( + str* attrs, + int enabled, + osips_malloc_f mf, + osips_free_f ff + ) +{ + trie_info_t* rt = NULL; + + rt = (trie_info_t*)func_malloc(mf, sizeof(trie_info_t)); + if (rt==NULL) { + LM_ERR("no more mem(1)\n"); + goto err_exit; + } + memset(rt, 0, sizeof(trie_info_t)); + rt->enabled = enabled; + + if (attrs && attrs->s && attrs->len) { + rt->attrs.s = func_malloc(mf,attrs->len); + if (rt->attrs.s == NULL) { + LM_ERR("no more shm mem(1)\n"); + goto err_exit; + } + rt->attrs.len = attrs->len; + memcpy(rt->attrs.s,attrs->s,rt->attrs.len); + } + + return rt; + +err_exit: + if (NULL!=rt->attrs.s) + func_free(ff,rt->attrs.s); + if ((NULL != rt) ) { + func_free(ff, rt); + } + return NULL; +} + + +void free_trie_data( + trie_data_t* rt_data, + osips_free_f free_f + ) +{ + if(NULL!=rt_data) { + del_tree(rt_data->pt, free_f); + rt_data->pt = 0 ; + + /* del top level */ + func_free(free_f, rt_data); + } +} diff --git a/modules/trie/prefix_tree.h b/modules/trie/prefix_tree.h new file mode 100644 index 00000000000..a9f773da983 --- /dev/null +++ b/modules/trie/prefix_tree.h @@ -0,0 +1,126 @@ + /* + * Trie Module + * + * Copyright (C) 2024 OpenSIPS Project + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * History: + * -------- + * 2024-12-03 initial release (vlad) + */ + +#ifndef trie_prefix_tree_h +#define trie_prefix_tree_h + +#include "../../str.h" +#include "../../ip_addr.h" +#include "../../time_rec.h" +#include "../../map.h" +#include "../../mem/mem_funcs.h" + +#define IS_DECIMAL_DIGIT(d) \ + (((d)>='0') && ((d)<= '9')) + +extern int ptree_children; +extern int tree_size; +struct head_db; + +#define INIT_TRIE_NODE(f, n) \ +do { \ + (n) = (trie_node_t*)func_malloc(f,sizeof(trie_node_t) + ptree_children * sizeof(trie_node_t*)); \ + if ((n) == NULL) \ + goto err_exit; \ + memset((n), 0, sizeof(trie_node_t) + ptree_children * sizeof(trie_node_t*)); \ +} while(0) + +#define SET_TRIE_CHILD(parent, child_index, new_node) \ +do { \ + trie_node_t **child_ptr = (trie_node_t**)(((char*)parent) + sizeof(trie_node_t) + child_index * sizeof(trie_node_t*)); \ + if (child_ptr != NULL) { \ + *child_ptr = new_node; /* Set the allocated node as the child */ \ + } \ +} while(0) + +typedef struct trie_info_ { + /* opaque string with rule attributes */ + str attrs; + /* enabled ? */ + int enabled; +} trie_info_t; + +typedef struct trie_node_ { + trie_info_t *info; + /* this node's children follow right after + * inside the initially allocated memory chunk + * use INIT_TRIE_NODE and SET_TRIE_CHILD for operating it */ +} trie_node_t; + +typedef struct trie_data_ { + /* tree with routing prefixes */ + trie_node_t *pt; +}trie_data_t; + +/* init new trie_data structure */ +trie_data_t* +build_trie_data( struct head_db * ); + + +void +free_trie_data(trie_data_t*, osips_free_f); + +int +init_prefix_tree( + char *extra_prefix_chars + ); + +int +del_tree( + trie_node_t *, + osips_free_f + ); + +int +add_trie_prefix( + trie_node_t*, + str* prefix, + trie_info_t *info, + osips_malloc_f, + osips_free_f + ); + +trie_info_t* +get_trie_prefix( + trie_node_t *ptree, + str* prefix, + unsigned int *matched_len, + int filter_disabled + ); + +trie_info_t* +build_trie_info( + str *attrs, + int disabled, + osips_malloc_f mf, + osips_free_f ff + ); + +void +free_trie_info( + trie_info_t*, + osips_free_f + ); + +#endif diff --git a/modules/trie/trie.c b/modules/trie/trie.c new file mode 100644 index 00000000000..9576073c6be --- /dev/null +++ b/modules/trie/trie.c @@ -0,0 +1,1292 @@ + /* + * Trie Module + * + * Copyright (C) 2024 OpenSIPS Project + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * History: + * -------- + * 2024-12-03 initial release (vlad) + */ + +#include +#include +#include +#include + + +#include "../../evi/evi.h" +#include "../../map.h" +#include "../../ipc.h" + +#include "trie_load.h" +#include "trie_db_def.h" +#include "prefix_tree.h" +#include "trie_partitions.h" + +#include "../../mem/rpm_mem.h" + +#define TRIE_PARAM_STRICT_LEN (1<<0) + +#define TRIE_TABLE_VER 1 +#define PART_TABLE_VER 1 + +#define MAX_LEN_NAME_W_PART 510 /* max len of variable containing + avp_spec and partition name */ +#define MI_PART_NAME_S "Partition" +#define MI_PART_NAME_LEN (strlen(MI_PART_NAME_S)) + +#define MI_LAST_UPDATE_S "Date" +#define MI_LAST_UPDATE_LEN (strlen(MI_LAST_UPDATE_S)) + +#define MI_LAST_DB_URL_S "DB_URL" +#define MI_LAST_DB_URL_LEN (strlen(MI_LAST_DB_URL_S)) + +#define MI_HASH_S "HASH" +#define MI_HASH_LEN (strlen(MI_HASH_S)) + +/* reload control parameter */ +static int no_concurrent_reload = 0; + +/* parameters */ +static str db_url = {NULL,0}; + +/* statistic data */ +int tree_size = 0; +static str attrs_empty = str_init(""); + +/* configuration loader from db specific stuff */ +static str trie_partitions_table = str_init("trie_partitions"); +static str trie_partitions_url; + +static str data_dump_folder = {0,0}; + +int use_partitions = 0; /* by default don't use db for config */ +static struct head_config { + str partition; /* partition name extracted from database */ + str db_url; + str trie_table; /* trie_table name extracted from database */ + struct head_config *next; +} *head_start; +int *n_partitions; /* total number of partitions (does not change at runtime) */ + +struct head_db *head_db_start; + +static int get_config_from_db(); +static int add_head_config(); +void init_head_db(struct head_db *new); +static int db_connect_head(struct head_db*); /* populate a db connection */ +static char *extra_prefix_chars; + + +/* reader-writers lock for reloading the data */ +static rw_lock_t *ref_lock = NULL; + +static int trie_init(void); +static int trie_child_init(int rank); +static int trie_exit(void); + +static int fix_flags(void** param); +static int fix_partition(void** param); + +static int trie_match(struct sip_msg* msg,str *number, long flags, + pv_spec_t* rule_att, pv_spec_t* match_prefix, struct head_db *part); + +mi_response_t *trie_reload_cmd(const mi_params_t *params, + struct mi_handler *async_hdl); +mi_response_t *trie_reload_cmd_1(const mi_params_t *params, + struct mi_handler *async_hdl); +mi_response_t *mi_trie_number_routing_1(const mi_params_t *params, + struct mi_handler *async_hdl); +mi_response_t *mi_trie_number_routing_2(const mi_params_t *params, + struct mi_handler *async_hdl); + +mi_response_t *mi_trie_reload_status(const mi_params_t *params, + struct mi_handler *async_hdl); +mi_response_t *mi_trie_reload_status_1(const mi_params_t *params, + struct mi_handler *async_hdl); +mi_response_t *mi_trie_remove_code_2(const mi_params_t *params,struct mi_handler *async_hdl); +mi_response_t *mi_trie_upsert_code_3(const mi_params_t *params,struct mi_handler *async_hdl); + +/* + * Exported functions + */ +static cmd_export_t cmds[] = { + {"trie_search", (cmd_function)trie_match, + { + {CMD_PARAM_STR, NULL, NULL}, + {CMD_PARAM_STR|CMD_PARAM_OPT, fix_flags, NULL}, + {CMD_PARAM_VAR|CMD_PARAM_OPT, NULL, NULL}, + {CMD_PARAM_VAR|CMD_PARAM_OPT, NULL, NULL}, + {CMD_PARAM_STR|CMD_PARAM_OPT|CMD_PARAM_FIX_NULL, fix_partition,NULL}, + {0 , 0, 0} + }, + REQUEST_ROUTE|FAILURE_ROUTE|LOCAL_ROUTE|BRANCH_ROUTE|ONREPLY_ROUTE + }, + {0,0,{{0,0,0}},0} +}; + + +/* + * Exported parameters + */ +static param_export_t params[] = { + {"use_partitions", INT_PARAM, &use_partitions }, + {"db_partitions_url", STR_PARAM, &trie_partitions_url.s }, + {"db_partitions_table", STR_PARAM, &trie_partitions_table.s }, + {"db_url", STR_PARAM, &db_url.s }, + {"trie_table", STR_PARAM, &trie_table.s }, + {"no_concurrent_reload",INT_PARAM, &no_concurrent_reload }, + {"extra_prefix_chars", STR_PARAM, &extra_prefix_chars }, + {0, 0, 0} +}; + + +/* + * Exported MI functions + */ +#define HLP1 "Params: none ; Forces trie module to reload data from DB "\ + "into memory; A return string is returned only in case of error." +#define HLP2 "Params: [partition] number ; Check if a "\ + "number will match when searching through the trie. "\ +"The partition parameter must be defined only if use_partitions = 1." +#define HLP3 "Params: [partition]; List the time of the last trie_reload"\ + " (load from database) for all partitions if no parameter is supplied, or"\ +" for a partition given as parameter. If use_partitions is 0, you should"\ +" not specify a partition." +#define HLP4 "Params: partitionid code_array ; Used to delete codes from the in-memory trie " +#define HLP5 "Params: partitionid code_array attrs_array ; Used to upsert codes in the in-memory trie" + +static mi_export_t mi_cmds[] = { + { "trie_reload", HLP1, 0, 0, { + {trie_reload_cmd, {0}}, + {trie_reload_cmd_1, {"partition_name", 0}}, + {EMPTY_MI_RECIPE}} + }, + { "trie_search", HLP2, MI_NAMED_PARAMS_ONLY, 0, { + {mi_trie_number_routing_1, {"number", 0}}, + {mi_trie_number_routing_2, {"partition_name", "number", 0}}, + {EMPTY_MI_RECIPE}} + }, + { "trie_reload_status", HLP3, 0, 0, { + {mi_trie_reload_status, {0}}, + {mi_trie_reload_status_1, {"partition_name", 0}}, + {EMPTY_MI_RECIPE}} + }, + { "trie_number_delete", HLP4, 0,0, { + {mi_trie_remove_code_2, {"partition_name","number",0}}, + {EMPTY_MI_RECIPE}} + }, + { "trie_number_upsert", HLP5, 0,0, { + {mi_trie_upsert_code_3, {"partition_name","number","attrs",0}}, + {EMPTY_MI_RECIPE}} + }, + {EMPTY_MI_EXPORT} +}; + +static dep_export_t deps = { + { /* OpenSIPS module dependencies */ + { MOD_TYPE_SQLDB, NULL, DEP_ABORT }, + { MOD_TYPE_NULL, NULL, 0 }, + }, + { /* modparam dependencies */ + { NULL, NULL }, + }, +}; + +struct module_exports exports = { + "trie", + MOD_TYPE_DEFAULT,/* class of this module */ + MODULE_VERSION, + DEFAULT_DLFLAGS, /* dlopen flags */ + 0, /* load function */ + &deps, /* OpenSIPS module dependencies */ + cmds, /* Exported functions */ + 0, /* Exported async functions */ + params, /* Exported parameters */ + 0, /* exported statistics */ + mi_cmds, /* exported MI functions */ + 0, /* exported pseudo-variables */ + 0, /* exported transformations */ + 0, /* additional processes */ + 0, /* Module pre-initialization function */ + trie_init, /* Module initialization function */ + (response_function) 0, + (destroy_function) trie_exit, + (child_init_function) trie_child_init, /* per-child init function */ + 0 /* reload confirm function */ +}; + +static void bin_hash_to_hex(HASH _b, HASHHEX _h) +{ + unsigned short i; + unsigned char j; + + for (i = 0; i < HASHLEN; i++) { + j = (_b[i] >> 4) & 0xf; + if (j <= 9) { + _h[i * 2] = (j + '0'); + } else { + _h[i * 2] = (j + 'a' - 10); + } + + j = _b[i] & 0xf; + + if (j <= 9) { + _h[i * 2 + 1] = (j + '0'); + } else { + _h[i * 2 + 1] = (j + 'a' - 10); + } + }; + + _h[HASHHEXLEN] = '\0'; +} + +/* + * if none is successfully loaded return + * -1, else return 0 + */ + +static inline int trie_reload_data_head(struct head_db *hd, + str *part_name, int initial) +{ + trie_data_t *new_data; + trie_data_t *old_data; + time_t rawtime; + MD5_CTX Md5Ctx; + HASH bin_md5; + FILE *fp=NULL; + + if (no_concurrent_reload) { + lock_get( hd->ref_lock->lock ); + if (hd->ongoing_reload) { + lock_release( hd->ref_lock->lock ); + LM_WARN("Reload already in progress, discarding this one\n"); + return -2; + } + hd->ongoing_reload = 1; + lock_release( hd->ref_lock->lock ); + } + + LM_INFO("loading trie data in partition %.*s\n",part_name->len,part_name->s); + MD5Init(&Md5Ctx); + + new_data = trie_load_info(hd, &Md5Ctx, fp); + if ( new_data==0 ) { + LM_CRIT("failed to load routing info\n"); + goto error; + } + + lock_start_write( hd->ref_lock ); + + /* no more activ readers -> do the swapping */ + old_data = hd->rdata; + hd->rdata = new_data; + /* update the time of the last reload for the current partition */ + time(&rawtime); + + hd->time_last_update = rawtime; + + MD5Final(bin_md5, &Md5Ctx); + bin_hash_to_hex(bin_md5,hd->md5); + + lock_stop_write( (hd->ref_lock) ); + + LM_INFO("loaded trie data in partition %.*s\n",part_name->len,part_name->s); + + /* destroy old data */ + if (old_data) { + /* free old data */ + free_trie_data(old_data, hd->free); + } + LM_INFO("destroyed old trie data in partition %.*s\n",part_name->len,part_name->s); + + if (no_concurrent_reload) + hd->ongoing_reload = 0; + return 0; + +error: + if (no_concurrent_reload) + hd->ongoing_reload = 0; + + return -1; +} + +static inline int trie_reload_data(int initial) +{ + struct head_db *part; + int ret_val = 0; + + for (part = head_db_start; part; part = part->next) + if (trie_reload_data_head(part, &part->partition, initial) != 0) + ret_val = -1; + + return ret_val; +} + +static int cleanup_head_config( struct head_config *hd) +{ + if (hd == NULL) + return 0; + + if (hd->db_url.s) + shm_free(hd->db_url.s); + if (hd->trie_table.s && hd->trie_table.s != trie_table.s) + shm_free(hd->trie_table.s); + + return 0; +} + + +static void cleanup_head_db(struct head_db *hd) +{ + if (!hd) + return; + + if (hd->db_con && *(hd->db_con)) + hd->db_funcs.close(*(hd->db_con)); + if( hd->ref_lock ) + lock_destroy_rw( ref_lock ); + if (hd->partition.s) + shm_free(hd->partition.s); + if (hd->db_url.s) + shm_free( hd->db_url.s ); + if (hd->trie_table.s && hd->trie_table.s != trie_table.s) + shm_free(hd->trie_table.s); +} + +static void cleanup_head_db_table(void) +{ + struct head_db * it_head_db = 0; + struct head_db * last_cleaned = 0; + + it_head_db = head_db_start; + while (it_head_db) { + + cleanup_head_db(it_head_db); + last_cleaned = it_head_db; + it_head_db = it_head_db->next; + shm_free(last_cleaned); + } + head_start = 0; +} + +static void cleanup_head_config_table(void) +{ + struct head_config * it_head_config = 0; + struct head_config * last_cleaned = 0; + + it_head_config = head_start; + while (it_head_config) { + + cleanup_head_config(it_head_config); + last_cleaned = it_head_config; + it_head_config = it_head_config->next; + shm_free(last_cleaned); + } + head_start = 0; +} + +static int trie_init(void) +{ + str name_w_part; + struct head_config * it_head_config = 0; + struct head_db *db_part = NULL; + char name_w_buf[MAX_LEN_NAME_W_PART]; + name_w_part.s = name_w_buf; + + if (data_dump_folder.s) + data_dump_folder.len = strlen(data_dump_folder.s); + + LM_INFO("trie - initializing\n"); + + n_partitions = shm_malloc(sizeof *n_partitions); + if (!n_partitions) { + LM_ERR("oom\n"); + return -1; + } + *n_partitions = 0; + + trie_table.len = strlen(trie_table.s); + + name_w_part.s = shm_malloc( MAX_LEN_NAME_W_PART ); + if( name_w_part.s == 0 ) { + LM_ERR(" No more shm memory [trie:name_w_part.s]\n"); + goto error; + } + + if( use_partitions == 1 ) { /* loading configurations from db */ + if (get_config_from_db() == -1) { + LM_ERR("Failed to get configuration from db_config\n"); + return -1; + } + } else { + init_db_url(db_url, 0); + + add_head_config(); + + /* if not empty save to head_config structure */ + if (trie_table.s[0]==0) { + LM_CRIT("mandatory parameter \"TRIE_TABLE\" found empty\n"); + goto error_cfg; + } + head_start->trie_table.s = shm_malloc(trie_table.len); + if (head_start->trie_table.s == 0) { + LM_ERR("no more shm memory [trie:head_start->trie_table.s]\n"); + goto error_cfg; + } + memcpy(head_start->trie_table.s, trie_table.s, trie_table.len); + head_start->trie_table.len = trie_table.len; + + head_start->db_url.len = db_url.len; + head_start->db_url.s = shm_malloc(db_url.len); + if( head_start->db_url.s == 0 ) { + LM_ERR("no more shm memory [trie:head_start->db_url.s]\n"); + goto error_cfg; + } + memcpy(head_start->db_url.s, db_url.s, db_url.len ); + + head_start->partition.s = "Default"; + head_start->partition.len = strlen(head_start->partition.s); + } + + if (init_prefix_tree( extra_prefix_chars )!=0) { + LM_ERR("failed to initiate the prefix array\n"); + goto error; + } + + for (it_head_config = head_start; it_head_config != NULL; + it_head_config = it_head_config->next) { + + db_part = shm_malloc(sizeof(struct head_db)); + if (!db_part) { + LM_ERR("could not allocate db part!\n"); + goto error_cfg; + } + init_head_db(db_part); + + if(shm_str_dup(&db_part->db_url, &it_head_config->db_url) != 0) { + LM_ERR("shm_str_dup failed for db_url\n"); + goto error_cfg; + } + + if(shm_str_dup(&db_part->partition, &it_head_config->partition) != 0) { + LM_ERR("shm_str_dup failed for partition name\n"); + goto error_cfg; + } + + if (!it_head_config->trie_table.s) { + db_part->trie_table.s = trie_table.s; + db_part->trie_table.len = trie_table.len; + } else if (shm_str_dup(&db_part->trie_table, &it_head_config->trie_table) != 0) { + LM_ERR("shm_str_dup failed for TRIE table\n"); + goto error_cfg; + } + + /* create & init lock */ + if ((db_part->ref_lock = lock_init_rw()) == NULL) { + LM_CRIT("failed to init lock\n"); + goto error_cfg; + } + + db_part->db_con = pkg_malloc(sizeof(db_con_t *)); + if (!db_part->db_con) { + LM_ERR("could not allocate db_connection in pkg mem!\n"); + goto error_cfg; + } + + /* bind to the SQL module */ + if (db_bind_mod( &(db_part->db_url), &( db_part->db_funcs ))) { + LM_CRIT("cannot bind to database module! " + "Did you forget to load a database module ? (%.*s)\n", + db_url.len, db_url.s); + goto error_cfg; + } + + if( (*db_part->db_con = + db_part->db_funcs.init(&db_part->db_url)) == 0) { + LM_ERR("failed to connect to db url <%.*s>\n", + db_part->db_url.len, db_part->db_url.s); + goto error_cfg; + } + + if (!DB_CAPABILITY( db_part->db_funcs, DB_CAP_QUERY)) { + LM_CRIT("database modules does not " + "provide QUERY functions needed by DRouting module\n"); + goto error_cfg; + } + + if(db_check_table_version(&db_part->db_funcs, *db_part->db_con, + &db_part->trie_table, TRIE_TABLE_VER) < 0) { + LM_ERR("error during table version check (trie table \'%.*s\'," + " for partition \'%.*s\')\n", db_part->trie_table.len, + db_part->trie_table.s, db_part->partition.len, + db_part->partition.s); + goto error_cfg; + } + + (db_part->db_funcs).close(*db_part->db_con); + *db_part->db_con = 0; + + /* all good now - add the partition to the list */ + db_part->next = head_db_start; + head_db_start = db_part; + db_part->malloc = shm_malloc_func; + db_part->free = shm_free_func; + } + /* all good now - release the config */ + cleanup_head_config_table(); + + LM_DBG("All in place in the init. Will return 0\n"); + return 0; + +error_cfg: + cleanup_head_config_table(); + if (db_part) { + cleanup_head_db(db_part); + shm_free(db_part); + } +error: + cleanup_head_db_table(); + return -1; +} + +static int db_connect_head(struct head_db *x) { + + if( *(x->db_con) ) { + LM_INFO("db_con already present\n"); + return 1; + } + if( x->db_url.s && (*(x->db_con) = x->db_funcs.init(&(x->db_url)))==0 ) { + LM_ERR("cannot initialize database connection" + "(partition:%.*s, db_url:%.*s, len:%d)\n", x->partition.len, + x->partition.s, x->db_url.len, x->db_url.s, x->db_url.len); + return -1; + } + return 0; +} + +/* simple wrapper over trie_reload_data to make it compatible with ipc_rpc_f, + * so triggerable via IPC */ +static void rpc_trie_reload_data(int sender_id, void *unused) +{ + trie_reload_data(1); +} + + +static int trie_child_init(int rank) +{ + struct head_db *db = head_db_start; + + LM_DBG("Child initialization on rank %d \n",rank); + + for (db = head_db_start; db; db = db->next) { + if (db_connect_head(db) < 0) { + LM_ERR("failed to create DB connection\n"); + return -1; + } + } + + /* if child 1, send a job for itself to run the data loading after + * the init sequance is done */ + if ( (rank==1) && ipc_send_rpc( process_no, rpc_trie_reload_data, NULL)<0) { + LM_CRIT("failed to RPC the data loading\n"); + return -1; + } + + return 0; +} + + +static int trie_exit(void) +{ + struct head_db * it = head_db_start, *to_clean; + + while( it!=NULL ) { + to_clean = it; + it = it->next; + + /* destroy data */ + if (to_clean->rdata) { + free_trie_data(to_clean->rdata, to_clean->free); + to_clean->rdata = 0; + } + + /* destroy lock */ + if (to_clean->ref_lock) { + lock_destroy_rw( to_clean->ref_lock ); + to_clean->ref_lock = 0; + } + + if(to_clean->trie_table.s && to_clean->trie_table.s != trie_table.s) { + shm_free(to_clean->trie_table.s); + } + + shm_free(to_clean); + } + + return 0; +} + +static mi_response_t *mi_trie_get_partition(const mi_params_t *params, + struct head_db **partition) +{ + str part_name; + + if (!use_partitions) + return init_mi_error_extra(400, + MI_SSTR("Invalid parameter: 'partition_name'"), + MI_SSTR("'partition_name' supported only when 'use_partitions' is set")); + + if (get_mi_string_param(params, "partition_name", + &part_name.s, &part_name.len) < 0) + return init_mi_param_error(); + + if((*partition = get_partition(&part_name)) == NULL) { + LM_ERR("Partition not found\n"); + return init_mi_error(404, MI_SSTR("Partition not found")); + } + + return NULL; +} + +mi_response_t *trie_reload_cmd(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + LM_INFO("trie_reload MI command received!\n"); + + if (trie_reload_data(0) != 0) { + LM_CRIT("failed to load routing data\n"); + return init_mi_error(500, MI_SSTR("Failed to reload")); + } + + return init_mi_result_ok(); +} + +mi_response_t *trie_reload_cmd_1(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + struct head_db *part; + mi_response_t *resp; + + LM_INFO("trie_reload MI command received!\n"); + + resp = mi_trie_get_partition(params, &part); + if (resp) + return resp; + + if (trie_reload_data_head(part, &part->partition, 0) < 0) { + LM_CRIT("Failed to load data head\n"); + return init_mi_error(500, MI_SSTR("Failed to reload")); + } + + return init_mi_result_ok(); +} + +struct head_db * get_partition(const str *name) +{ + struct head_db * it = head_db_start; + + while( it!= NULL) { + if( it->partition.len==name->len && memcmp( it->partition.s, name->s, + name->len)==0 ) { + return it; + } + it = it->next; + } + + return NULL; /* partition was not found */ +} + +static int fix_flags(void** param) +{ + str *s = (str*)(*param); + char *p; + long flags=0; + + if (s) { + for ( p=s->s ; ps+s->len ; p++ ) { + switch (*p) { + case 'L': + flags |= TRIE_PARAM_STRICT_LEN; + LM_DBG("matching prefix with strict len\n"); + break; + default: + LM_DBG("unknown flag : [%c] . Skipping\n",*p); + } + } + *param = (void*)(long)flags; + } + return 0; +} + + +static int fix_partition(void** param) +{ + str *s = (str*)(*param); + struct head_db *part; + + if (s==NULL) { + /* no partition defined */ + if (use_partitions==0) { + if(head_db_start == NULL) { + LM_ERR("Bad configuration, missing default partition\n"); + return -1; + } + part = head_db_start; + } else { + LM_ERR("Partition name is mandatory\n"); + return -1; + } + } else { + /* partition name defined */ + if (s->len==1 && s->s[0]=='*') { + /* partition wild card */ + part = NULL; + } else { + part = get_partition( s ); + if (part==NULL) { + LM_ERR("partition <%.*s> used, but not defined\n",s->len,s->s); + return -1; + } + } + } + *param = (void*)part; + + return 0; +} + +static int trie_match(struct sip_msg* msg, str *number,long flags, + pv_spec_t* rule_att, pv_spec_t* match_prefix, struct head_db *part) +{ + trie_info_t* rule; + unsigned int matched_len; + pv_value_t val; + + if (part==NULL || part->rdata == 0) + return -1; + + lock_start_read( part->ref_lock ); + + rule = get_trie_prefix(part->rdata->pt,number, &matched_len, 1); + if (rule == NULL){ + goto failure; + } + + /* was it a full prefix matching ? */ + if (flags & TRIE_PARAM_STRICT_LEN) { + if (matched_len!=number->len) + goto failure; + } + + if (rule_att) { + val.flags = PV_VAL_STR; + val.rs = !rule->attrs.s ? attrs_empty : rule->attrs; + if (pv_set_value(msg, rule_att, 0, &val) != 0) { + LM_ERR("failed to set value for rule attrs pvar\n"); + goto failure; + } + } + + /* add RULE prefix avp */ + if (match_prefix) { + val.flags = PV_VAL_STR; + val.rs.s = number->s; + val.rs.len = matched_len; + if (pv_set_value(msg, match_prefix, 0, &val) != 0) { + LM_ERR("failed to set value for rule attrs pvar\n"); + goto failure; + } + } + + lock_stop_read( part->ref_lock ); + + return 1; + +failure: + lock_stop_read( part->ref_lock ); + return -1; +} + +void init_head_db(struct head_db *new) +{ + memset(new, 0, sizeof(struct head_db)); +} + +/* use_partitions: use configurations from database */ +int add_head_config(void) +{ + /* expand linked list */ + struct head_config *new; + + new = shm_malloc(sizeof(struct head_config)); + if( new == NULL ) { + LM_ERR("no more shm memory\n"); + return -1; + } + memset(new, 0, sizeof(struct head_config)); + + new->next = head_start; + head_start = new; + + (*n_partitions)++; + return 0; +} + +#define init_head_config_value( from_head, external, default_val)\ + if( external.len!=0 ) {\ + shm_str_dup( &(from_head), &(external));\ + } else {\ + from_head = default_val;\ + }\ + +static int populate_head_config(struct head_config *current, str attr, int index) { + switch(index) { + case 0: + if(shm_str_dup( &(current->partition), &attr) < 0) { + LM_ERR("no more shm memory for partition_name in head_config\n"); + } + break; + case 1: + if( shm_str_dup(&(current->db_url), &attr) < 0) { + LM_ERR("no more shm memory for db_url in head_config\n"); + } + break; + case 2: + init_head_config_value( current->trie_table, attr, trie_table); + break; + default: + LM_DBG("Column from db_config not_known\n"); + return -1; + } + return 0; +} +static int get_config_from_db(void) { + + db_func_t db_funcs; + db_res_t * query_res; + db_con_t * db_con = 0; + /* columns needed from db_confgir_url for query */ + str partition_col = str_init("partition_name"); + str db_url_col = str_init("db_url"); + str table_col = str_init("trie_table"); + int n_query_col = 4; + db_key_t query_cols[] = {&partition_col, &db_url_col, &table_col}; + /* query result processing stuff */ + int nr_rows_db_config = 0 ; + int nr_cols_db_config = 0 ; + db_val_t * value; + db_row_t *rows_db_config = NULL; + int j; + int i; + str ans_col = {NULL, 0}; + + init_db_url(trie_partitions_url, 0); + trie_partitions_url.len = strlen(trie_partitions_url.s); + trie_partitions_table.len = strlen(trie_partitions_table.s); + + if(db_bind_mod( &trie_partitions_url, &db_funcs) < 0) { + LM_ERR("Unable to bind to database driver (partition definitions) " + "\n", trie_partitions_url.len, + trie_partitions_url.s); + goto error; + } + + if( (db_con = db_funcs.init(&trie_partitions_url)) == 0 ) { + LM_ERR("Cannot init connection to partitions table " + "\n", trie_partitions_url.len, + trie_partitions_url.s); + goto error; + } + + + if(db_check_table_version(&db_funcs, db_con, + &trie_partitions_table, PART_TABLE_VER) < 0) { + LM_ERR("error during table version check .\n", + trie_partitions_table.len, trie_partitions_table.s); + return -1; + } + + if( db_funcs.use_table( db_con, &trie_partitions_table) < 0) { + LM_ERR("Cannot use the partitions table " + "\n", trie_partitions_table.len, trie_partitions_table.s, + trie_partitions_url.len, trie_partitions_url.s); + goto error; + } + + /* query for populating head_config structure */ + if( db_funcs.query( db_con, NULL, NULL, NULL, query_cols, 0, n_query_col, + NULL, &query_res) < 0 ) { + LM_ERR("Failed to query the table containing the partition definitions " + "\n", + trie_partitions_url.len, trie_partitions_url.s, + trie_partitions_table.len, trie_partitions_table.s); + goto error; + } + + nr_rows_db_config = RES_ROW_N(query_res); + nr_cols_db_config = RES_COL_N(query_res); + rows_db_config = RES_ROWS(query_res); + + LM_DBG("Got %d total trie partitions \n",nr_rows_db_config); + + for( i=0; irdata == 0) + return init_mi_result_ok(); + + lock_start_read( partition->ref_lock ); + + if (partition->rdata == NULL) { + lock_stop_read( partition->ref_lock ); + return init_mi_error(400, MI_SSTR("No data")); + } + + + route = get_trie_prefix(partition->rdata->pt,&number,&matched_len, 1); + LM_DBG("Got back %p \n",route); + if (route == NULL){ + lock_stop_read( partition->ref_lock ); + return init_mi_result_string(MI_SSTR("No match")); + } + + resp = init_mi_result_object(&resp_obj); + if (!resp) + return 0; + + if (add_mi_string(resp_obj, MI_SSTR("Matched Prefix"), + number.s, matched_len) < 0) + goto error; + + if (route->attrs.s != NULL && route->attrs.len > 0) + if (add_mi_string(resp_obj, MI_SSTR("ATTRS"), + route->attrs.s,route->attrs.len) < 0) + goto error; + + lock_stop_read( partition->ref_lock ); + + return resp; + +error: + lock_stop_read( partition->ref_lock ); + free_mi_response(resp); + return 0; +} + +mi_response_t *mi_trie_number_routing_1(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + if (use_partitions) + return init_mi_error_extra(400, + MI_SSTR("Missing parameter: 'partition_name'"), + MI_SSTR("'partition_name' is required when 'use_partitions' is set")); + + return mi_trie_number_routing(params, head_db_start); +} + +mi_response_t *mi_trie_number_routing_2(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + struct head_db * current_partition=0; + mi_response_t *resp; + + resp = mi_trie_get_partition(params, ¤t_partition); + if (resp) + return resp; + + return mi_trie_number_routing(params, current_partition); +} + +static int mi_trie_print_rld_status(mi_item_t *part_item, struct head_db * partition, + int with_name) +{ + char ch_time[26]; + + lock_start_read(partition->ref_lock); + + ctime_r(&partition->time_last_update, ch_time); + LM_DBG("partition %.*s was last updated:%s\n", + partition->partition.len, partition->partition.s, + ch_time); + + if (with_name && add_mi_string(part_item, MI_SSTR("name"), + partition->partition.s, partition->partition.len) < 0) + goto error; + + if (add_mi_string(part_item, MI_SSTR(MI_LAST_UPDATE_S), + ch_time, strlen(ch_time)-1) < 0) + goto error; + + if (add_mi_string(part_item,MI_SSTR(MI_HASH_S),partition->md5,strlen(partition->md5)) < 0) + goto error; + + lock_stop_read(partition->ref_lock); + + return 0; + +error: + lock_stop_read(partition->ref_lock); + return -1; +} + +mi_response_t *mi_trie_reload_status(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + struct head_db * partition; + mi_response_t *resp; + mi_item_t *resp_obj; + mi_item_t *parts_arr, *part_item; + + resp = init_mi_result_object(&resp_obj); + if (!resp) + return 0; + + if(use_partitions){ + /* display for all partitions */ + parts_arr = add_mi_array(resp_obj, MI_SSTR("Partitions")); + if (!parts_arr) + goto error; + + for(partition = head_db_start; partition; partition = partition->next) { + part_item = add_mi_object(parts_arr, NULL, 0); + if (!part_item) + goto error; + + if (mi_trie_print_rld_status(part_item, partition, 1) < 0) + goto error; + } + } else /* just one partition */ + if (mi_trie_print_rld_status(resp_obj, head_db_start, 0) < 0) + goto error; + + return resp; + +error: + free_mi_response(resp); + return 0; +} + +mi_response_t *mi_trie_reload_status_1(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + struct head_db * partition; + mi_response_t *resp; + mi_item_t *resp_obj; + + resp = mi_trie_get_partition(params, &partition); + if (resp) + return resp; + + resp = init_mi_result_object(&resp_obj); + if (!resp) + return 0; + + if (mi_trie_print_rld_status(resp_obj, partition, 1) < 0) { + free_mi_response(resp); + return 0; + } + + return resp; +} + +mi_response_t *mi_trie_remove_code_2(const mi_params_t *params,struct mi_handler *async_hdl) +{ + struct head_db *partition; + str number; + unsigned int matched_len; + trie_info_t *route; + mi_response_t *resp; + mi_item_t *code_arr; + int no_codes,i; + + resp = mi_trie_get_partition(params,&partition); + if (resp) + return resp; + + if (get_mi_array_param(params, "number", &code_arr, &no_codes) < 0) + return init_mi_param_error(); + + lock_start_read( partition->ref_lock ); + + if (partition->rdata == NULL) { + lock_stop_read( partition->ref_lock ); + return init_mi_error(400, MI_SSTR("No data")); + } + + for (i = 0; i < no_codes; i++) { + if (get_mi_arr_param_string(code_arr, i, + &number.s, &number.len) < 0) { + lock_stop_read( partition->ref_lock ); + return init_mi_param_error(); + } + + route = get_trie_prefix(((partition->rdata))->pt, + &number, &matched_len,1); + if (route == NULL) { + LM_ERR("Failed to find DID to delete [%.*s]\n",number.len,number.s); + continue; + } + + if (matched_len != number.len) { + LM_ERR("Failed to find entry to delete [%.*s]\n",number.len,number.s); + continue; + } + + route->enabled = 0; + } + + lock_stop_read( partition->ref_lock ); + + return init_mi_result_ok(); +} + +mi_response_t *mi_trie_upsert_code_3(const mi_params_t *params,struct mi_handler *async_hdl) +{ + struct head_db *partition; + str number,attr,dyn_attr; + unsigned int matched_len; + trie_info_t *route; + mi_response_t *resp; + mi_item_t *code_arr, *attrs_arr; + int no_codes,no_attrs,i; + + resp = mi_trie_get_partition(params,&partition); + if (resp) + return resp; + + if (get_mi_array_param(params, "number", &code_arr, &no_codes) < 0) + return init_mi_param_error(); + if (get_mi_array_param(params, "attrs", &attrs_arr, &no_attrs) < 0) + return init_mi_param_error(); + + if (no_codes != no_attrs) { + return init_mi_error(400, MI_SSTR("Code attrs missmatch")); + } + + lock_start_read( partition->ref_lock ); + if (partition->rdata == NULL) { + lock_stop_read( partition->ref_lock ); + return init_mi_error(400, MI_SSTR("No data")); + } + + for (i = 0; i < no_codes; i++) { + if (get_mi_arr_param_string(code_arr, i, + &number.s, &number.len) < 0) { + lock_stop_read( partition->ref_lock ); + return init_mi_param_error(); + } + + if (get_mi_arr_param_string(attrs_arr, i, + &attr.s, &attr.len) < 0) { + lock_stop_read( partition->ref_lock ); + return init_mi_param_error(); + } + + /* we search for all codes, enabled or disabled */ + route = get_trie_prefix(((partition->rdata))->pt, + &number, &matched_len,0); + + if (matched_len != number.len){ + /* prefix not found, need to add it */ + + route = build_trie_info(&attr,1,partition->malloc,partition->free); + if (!route) { + LM_ERR("Failed to build route info for DID upsert %.*s\n", number.len,number.s); + lock_stop_read( partition->ref_lock ); + return init_mi_error(500, MI_SSTR("Internal Error")); + } + + if (add_trie_prefix(((partition->rdata))->pt,&number,route,partition->malloc,partition->free) != 0) { + LM_ERR("Failed to add route info for DID upsert %.*s\n", number.len,number.s); + lock_stop_read( partition->ref_lock ); + free_trie_info(route,partition->free); + return init_mi_error(500, MI_SSTR("Internal Error")); + } + } else { + /* we found it, need to update in-place */ + dyn_attr.s = shm_malloc(attr.len); + if (!dyn_attr.s) { + LM_ERR("No more shm \n"); + lock_stop_read( partition->ref_lock ); + return init_mi_error(500, MI_SSTR("Internal Error")); + } + + memcpy(dyn_attr.s,attr.s,attr.len); + if (route->attrs.s) { + shm_free(route->attrs.s); + } + + route->attrs.len = attr.len; + route->attrs.s = dyn_attr.s; + + /* if it was removed before, clear that flag */ + route->enabled = 1; + } + } + + lock_stop_read( partition->ref_lock ); + return init_mi_result_ok(); +} diff --git a/modules/trie/trie_db_def.c b/modules/trie/trie_db_def.c new file mode 100644 index 00000000000..13a26351505 --- /dev/null +++ b/modules/trie/trie_db_def.c @@ -0,0 +1,36 @@ + /* + * Trie Module + * + * Copyright (C) 2024 OpenSIPS Project + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * History: + * -------- + * 2024-12-03 initial release (vlad) + */ + +#include "../../ut.h" +#include "trie_db_def.h" + +/* DR rule table related defs */ +#define PREFIX_TRIE_COL "prefix" +#define ATTRS_TRIE_COL "attrs" +#define DISABLED_TRIE_COL "enabled" + +str trie_table = str_init("trie"); +str prefix_trie_col = str_init(PREFIX_TRIE_COL); +str attrs_trie_col = str_init(ATTRS_TRIE_COL); +str enabled_trie_col = str_init(DISABLED_TRIE_COL); diff --git a/modules/trie/trie_db_def.h b/modules/trie/trie_db_def.h new file mode 100644 index 00000000000..7940913fff0 --- /dev/null +++ b/modules/trie/trie_db_def.h @@ -0,0 +1,37 @@ + /* + * Trie Module + * + * Copyright (C) 2024 OpenSIPS Project + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * History: + * -------- + * 2024-12-03 initial release (vlad) + */ + +#ifndef _TRIE_DB_DEFS +#define _TRIE_DB_DEFS + +#include "../../str.h" + +/* DR rule table related defs */ +extern str trie_table; +extern str prefix_trie_col; +extern str attrs_trie_col; +extern str enabled_trie_col; + +#endif + diff --git a/modules/trie/trie_load.c b/modules/trie/trie_load.c new file mode 100644 index 00000000000..d3125382b5e --- /dev/null +++ b/modules/trie/trie_load.c @@ -0,0 +1,248 @@ + /* + * Trie Module + * + * Copyright (C) 2024 OpenSIPS Project + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * History: + * -------- + * 2024-12-03 initial release (vlad) + */ + +#include +#include +#include +#include +#include + + +#include "../../dprint.h" +#include "../../route.h" +#include "../../db/db.h" +#include "../../mem/shm_mem.h" +#include "../../mem/rpm_mem.h" +#include "../../time_rec.h" +#include "../../socket_info.h" + +#include "trie_load.h" +#include "prefix_tree.h" +#include "trie_db_def.h" + + +#define check_val2( _col, _val, _type1, _type2, _not_null, _is_empty_str) \ + do{\ + if ((_val)->type!=_type1 && (_val)->type!=_type2) { \ + LM_ERR("column %.*s has a bad type [%d], accepting only [%d,%d]\n",\ + _col.len, _col.s, (_val)->type, _type1, _type2); \ + goto error;\ + } \ + if (_not_null && (_val)->nul) { \ + LM_ERR("column %.*s is null\n", _col.len, _col.s); \ + goto error;\ + } \ + if (_is_empty_str && VAL_STRING(_val)==0) { \ + LM_ERR("column %.*s (str) is empty\n", _col.len, _col.s); \ + goto error;\ + } \ + }while(0) + +#define check_val( _col, _val, _type, _not_null, _is_empty_str) \ + do{\ + if ((_val)->type!=_type) { \ + LM_ERR("column %.*s has a bad type [%d], accepting only [%d]\n",\ + _col.len, _col.s, (_val)->type, _type); \ + goto error;\ + } \ + if (_not_null && (_val)->nul) { \ + LM_ERR("column %.*s is null\n", _col.len, _col.s); \ + goto error;\ + } \ + if (_is_empty_str && VAL_STRING(_val)==0) { \ + LM_ERR("column %.*s (str) is empty\n", _col.len, _col.s); \ + goto error;\ + } \ + }while(0) + + +void hash_rule(str* prefix, trie_info_t* rule, MD5_CTX* hash_ctx, FILE* fp) +{ + if (prefix->s && prefix->len) { + MD5Update(hash_ctx, prefix->s, prefix->len); + if (fp) + fprintf(fp, " %.*s",prefix->len,prefix->s); + } + + if (rule->attrs.s && rule->attrs.len) { + MD5Update(hash_ctx, rule->attrs.s, rule->attrs.len); + if (fp) + fprintf(fp, " %.*s",rule->attrs.len,rule->attrs.s); + } + + if (fp) + fprintf(fp,"\n"); +} + +static int add_rule(trie_data_t *rdata, str *prefix, + trie_info_t *rule, osips_malloc_f malloc_f, osips_free_f free_f,MD5_CTX *hash_ctx, FILE *fp) +{ + if ( add_trie_prefix(rdata->pt, prefix, rule,malloc_f, free_f)!=0 ) { + LM_ERR("failed to add prefix route\n"); + goto error; + } + + hash_rule(prefix,rule,hash_ctx,fp); + + return 0; +error: + return -1; +} + +/* loads trie info for given partition; if partition_name is NULL + * loads all partitions + */ + +trie_data_t* trie_load_info(struct head_db *current_partition, MD5_CTX* hash_ctx, FILE *fp) +{ + db_func_t *trie_dbf; + db_con_t* db_hdl; + str *trie_table = ¤t_partition->trie_table; + db_key_t columns[3]; + db_res_t* res; + db_row_t* row; + trie_info_t *ri; + trie_data_t *rdata; + int i,n; + int no_rows = 10; + str prefix,attrs; + + trie_dbf = ¤t_partition->db_funcs; + db_hdl = *current_partition->db_con; + + res = 0; + ri = 0; + rdata = 0; + + /* init new data structure */ + if ( (rdata=build_trie_data(current_partition))==0 ) { + LM_ERR("failed to build rdata\n"); + goto error; + } + + + /* read the routing rules */ + if (trie_dbf->use_table( db_hdl, trie_table) < 0) { + LM_ERR("cannot select table \"%.*s\"\n", trie_table->len, trie_table->s); + goto error; + } + + columns[0] = &prefix_trie_col; + columns[1] = &attrs_trie_col; + columns[2] = &enabled_trie_col; + + if (DB_CAPABILITY(*trie_dbf, DB_CAP_FETCH)) { + if ( trie_dbf->query( db_hdl, 0, 0, 0, columns, 0, 3, 0, 0) < 0) { + LM_ERR("DB query failed\n"); + goto error; + } + no_rows = estimate_available_rows( 32+128+4, 3/*cols*/); + if (no_rows==0) no_rows = 10; + if(trie_dbf->fetch_result(db_hdl, &res, no_rows)<0) { + LM_ERR("Error fetching rows\n"); + goto error; + } + } else { + if ( trie_dbf->query( db_hdl, 0, 0, 0, columns, 0, 3, 0, &res) < 0) { + LM_ERR("DB query failed\n"); + goto error; + } + } + + if (RES_ROW_N(res) == 0) { + LM_WARN("table \"%.*s\" is empty\n", trie_table->len, trie_table->s); + } + + LM_DBG("initial %d records found in %.*s\n", RES_ROW_N(res), + trie_table->len, trie_table->s); + + n = 0; + do { + for(i=0; i < RES_ROW_N(res); i++) { + row = RES_ROWS(res) + i; + /* PREFIX column */ + check_val( prefix_trie_col, ROW_VALUES(row), DB_STRING, 1, 1); + + prefix.s = (char *)VAL_STRING(ROW_VALUES(row)); + prefix.len = strlen(prefix.s); + + if ((ROW_VALUES(row)+1)->nul || VAL_STRING(ROW_VALUES(row)+1) == NULL) { + attrs.s = NULL; + attrs.len = 0; + } else { + attrs.s = (char *)VAL_STRING(ROW_VALUES(row)+1); + attrs.len = strlen(attrs.s); + } + + LM_DBG("Fetched %.*s prefix \n",VAL_STR(ROW_VALUES(row)).len,VAL_STR(ROW_VALUES(row)).s); + + /* build the routing rule */ + if ((ri = build_trie_info( + &attrs, + VAL_INT(ROW_VALUES(row)+2), + current_partition->malloc, + current_partition->free))== 0 ) { + LM_ERR("failed to add routing info for rule prefix %.*s\n", VAL_STR(ROW_VALUES(row)+1).len,VAL_STR(ROW_VALUES(row)+1).s); + continue; + } + /* add the rule */ + if (add_rule( + rdata, + &prefix, + ri, + current_partition->malloc, + current_partition->free,hash_ctx, fp)!=0) { + + LM_ERR("failed to add routing info for rule prefix %.*s\n", VAL_STR(ROW_VALUES(row)+1).len,VAL_STR(ROW_VALUES(row)+1).s); + free_trie_info(ri, current_partition->free); + continue; + } + n++; + } + if (DB_CAPABILITY(*trie_dbf, DB_CAP_FETCH)) { + if(trie_dbf->fetch_result(db_hdl, &res, no_rows)<0) { + LM_ERR( "fetching rows (1)\n"); + goto error; + } + LM_DBG("additional %d records found in %.*s\n", RES_ROW_N(res), + trie_table->len, trie_table->s); + } else { + break; + } + } while(RES_ROW_N(res)>0); + + trie_dbf->free_result(db_hdl, res); + res = 0; + + LM_DBG("%d total records loaded from table %.*s\n", n, + trie_table->len, trie_table->s); + return rdata; +error: + if (res) + trie_dbf->free_result(db_hdl, res); + if (rdata) + free_trie_data(rdata, current_partition->free); + rdata = NULL; + return 0; +} diff --git a/modules/trie/trie_load.h b/modules/trie/trie_load.h new file mode 100644 index 00000000000..3c8da7a6337 --- /dev/null +++ b/modules/trie/trie_load.h @@ -0,0 +1,34 @@ + /* + * Trie Module + * + * Copyright (C) 2024 OpenSIPS Project + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * History: + * -------- + * 2024-12-03 initial release (vlad) + */ + +#ifndef _TRIE_LOAD_ +#define _TRIE_LOAD_ + +#include "../../str.h" +#include "../../db/db.h" +#include "trie_partitions.h" + +trie_data_t* trie_load_info(struct head_db *current_partition, MD5_CTX *hash_ctx, FILE* fp); + +#endif diff --git a/modules/trie/trie_partitions.h b/modules/trie/trie_partitions.h new file mode 100644 index 00000000000..be8f6e75b64 --- /dev/null +++ b/modules/trie/trie_partitions.h @@ -0,0 +1,67 @@ + /* + * Trie Module + * + * Copyright (C) 2024 OpenSIPS Project + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * History: + * -------- + * 2024-12-03 initial release (vlad) + */ + +#ifndef TRIE_PARTITIONS_H +#define TRIE_PARTITIONS_H + +#include "prefix_tree.h" +#include "../../db/db.h" +#include "../../mem/mem.h" +#include "../../mem/shm_mem.h" +#include "../../rw_locking.h" +#include "../../action.h" +#include "../../error.h" +#include "../../ut.h" +#include "../../mod_fix.h" +#include "../../md5global.h" +#include "../../md5.h" + +extern int use_partitions; +extern rw_lock_t *reload_lock; + +#define HASHLEN 16 +typedef char HASH[HASHLEN]; + +#define HASHHEXLEN 32 +typedef char HASHHEX[HASHHEXLEN+1]; + +struct head_db { + str db_url; + str partition; + db_func_t db_funcs; + db_con_t **db_con; + str trie_table; /* trie_table name extracted from database */ + time_t time_last_update; + trie_data_t *rdata; + HASHHEX md5; + rw_lock_t *ref_lock; + int ongoing_reload; + struct head_db *next; + osips_malloc_f malloc; + osips_free_f free; +}; + +struct head_db * get_partition(const str *); + +#endif diff --git a/scripts/db_berkeley/opensips/dr_partitions b/scripts/db_berkeley/opensips/dr_partitions index b826ecadda9..31cc7518838 100644 --- a/scripts/db_berkeley/opensips/dr_partitions +++ b/scripts/db_berkeley/opensips/dr_partitions @@ -1,5 +1,5 @@ METADATA_COLUMNS -id(int) partition_name(str) db_url(str) drd_table(str) drr_table(str) drg_table(str) drc_table(str) ruri_avp(str) gw_id_avp(str) gw_priprefix_avp(str) gw_sock_avp(str) rule_id_avp(str) rule_prefix_avp(str) carrier_id_avp(str) +id(int) partition_name(str) db_url(str) trie_table(str) METADATA_KEY 0 METADATA_READONLY @@ -7,4 +7,4 @@ METADATA_READONLY METADATA_LOGFLAGS 0 METADATA_DEFAULTS -NIL|NIL|NIL|NIL|NIL|NIL|NIL|NIL|NIL|NIL|NIL|NIL|NIL|NIL +NIL|NIL|NIL|NIL diff --git a/scripts/db_berkeley/opensips/trie_table b/scripts/db_berkeley/opensips/trie_table new file mode 100644 index 00000000000..8c24bac1f78 --- /dev/null +++ b/scripts/db_berkeley/opensips/trie_table @@ -0,0 +1,12 @@ +METADATA_COLUMNS +ruleid(int) prefix(str) attrs(str) priority(int) +METADATA_KEY +0 +METADATA_READONLY +0 +METADATA_LOGFLAGS +0 +METADATA_DEFAULTS +NIL|NIL|NULL|1 +trie_table| +trie_table|1 diff --git a/scripts/dbtext/opensips/dr_partitions b/scripts/dbtext/opensips/dr_partitions index da439109c38..8e304ea073e 100644 --- a/scripts/dbtext/opensips/dr_partitions +++ b/scripts/dbtext/opensips/dr_partitions @@ -1 +1 @@ -id(int,auto) partition_name(string) db_url(string) drd_table(string,null) drr_table(string,null) drg_table(string,null) drc_table(string,null) ruri_avp(string,null) gw_id_avp(string,null) gw_priprefix_avp(string,null) gw_sock_avp(string,null) rule_id_avp(string,null) rule_prefix_avp(string,null) carrier_id_avp(string,null) +id(int,auto) partition_name(string) db_url(string) trie_table(string,null) diff --git a/scripts/dbtext/opensips/trie_table b/scripts/dbtext/opensips/trie_table new file mode 100644 index 00000000000..5812bc57d19 --- /dev/null +++ b/scripts/dbtext/opensips/trie_table @@ -0,0 +1,2 @@ +ruleid(int,auto) prefix(string) attrs(string,null) priority(int) +trie_table:1 diff --git a/scripts/mysql/trie-create.sql b/scripts/mysql/trie-create.sql new file mode 100644 index 00000000000..e1edab30e00 --- /dev/null +++ b/scripts/mysql/trie-create.sql @@ -0,0 +1,16 @@ +INSERT INTO version (table_name, table_version) values ('trie_table','1'); +CREATE TABLE trie_table ( + ruleid INT(10) UNSIGNED AUTO_INCREMENT PRIMARY KEY NOT NULL, + prefix CHAR(64) NOT NULL, + attrs CHAR(255) DEFAULT NULL, + priority INT(11) DEFAULT 1 NOT NULL +) ENGINE=InnoDB; + +INSERT INTO version (table_name, table_version) values ('dr_partitions','1'); +CREATE TABLE dr_partitions ( + id INT(10) UNSIGNED AUTO_INCREMENT PRIMARY KEY NOT NULL, + partition_name CHAR(255) NOT NULL, + db_url CHAR(255) NOT NULL, + trie_table CHAR(255) +) ENGINE=InnoDB; + diff --git a/scripts/oracle/trie-create.sql b/scripts/oracle/trie-create.sql new file mode 100644 index 00000000000..82de8054747 --- /dev/null +++ b/scripts/oracle/trie-create.sql @@ -0,0 +1,32 @@ +INSERT INTO version (table_name, table_version) values ('trie_table','1'); +CREATE TABLE trie_table ( + ruleid NUMBER(10) PRIMARY KEY, + prefix VARCHAR2(64), + attrs VARCHAR2(255) DEFAULT NULL, + priority NUMBER(10) DEFAULT 1 NOT NULL +); + +CREATE OR REPLACE TRIGGER trie_table_tr +before insert on trie_table FOR EACH ROW +BEGIN + auto_id(:NEW.id); +END trie_table_tr; +/ +BEGIN map2users('trie_table'); END; +/ +INSERT INTO version (table_name, table_version) values ('dr_partitions','1'); +CREATE TABLE dr_partitions ( + id NUMBER(10) PRIMARY KEY, + partition_name VARCHAR2(255), + db_url VARCHAR2(255), + trie_table VARCHAR2(255) +); + +CREATE OR REPLACE TRIGGER dr_partitions_tr +before insert on dr_partitions FOR EACH ROW +BEGIN + auto_id(:NEW.id); +END dr_partitions_tr; +/ +BEGIN map2users('dr_partitions'); END; +/ diff --git a/scripts/pi_http/pi_framework.xml b/scripts/pi_http/pi_framework.xml index 58963803caf..eeba67c984d 100644 --- a/scripts/pi_http/pi_framework.xml +++ b/scripts/pi_http/pi_framework.xml @@ -1044,6 +1044,24 @@ fromtagDB_STR directionDB_STR + + + trie_table + mysql + ruleidDB_INT + prefixDB_STR + attrsDB_STR + priorityDB_INT + + + + dr_partitions + mysql + idDB_INT + partition_nameDB_STR + db_urlDB_STR + trie_tableDB_STR + userblacklist @@ -5067,6 +5085,88 @@ + + trie_table + show + trie_table + DB_QUERY + + ruleidupdate + prefix + attrs + priority + + + add + trie_table + DB_INSERT + + prefix + attrs + priority + + + update + trie_table + DB_UPDATE + + ruleid= + + + prefix + attrs + priority + + + delete + trie_table + DB_DELETE + + ruleid= + + + + + dr_partitions + show + dr_partitions + DB_QUERY + + idupdate + partition_name + db_url + trie_table + + + add + dr_partitions + DB_INSERT + + partition_name + db_url + trie_table + + + update + dr_partitions + DB_UPDATE + + id= + + + partition_name + db_url + trie_table + + + delete + dr_partitions + DB_DELETE + + id= + + + userblacklist show diff --git a/scripts/pi_http/trie-mod b/scripts/pi_http/trie-mod new file mode 100644 index 00000000000..5fe4e835e59 --- /dev/null +++ b/scripts/pi_http/trie-mod @@ -0,0 +1,82 @@ + + trie_table + show + trie_table + DB_QUERY + + ruleidupdate + prefix + attrs + priority + + + add + trie_table + DB_INSERT + + prefix + attrs + priority + + + update + trie_table + DB_UPDATE + + ruleid= + + + prefix + attrs + priority + + + delete + trie_table + DB_DELETE + + ruleid= + + + + + dr_partitions + show + dr_partitions + DB_QUERY + + idupdate + partition_name + db_url + trie_table + + + add + dr_partitions + DB_INSERT + + partition_name + db_url + trie_table + + + update + dr_partitions + DB_UPDATE + + id= + + + partition_name + db_url + trie_table + + + delete + dr_partitions + DB_DELETE + + id= + + + diff --git a/scripts/pi_http/trie-table b/scripts/pi_http/trie-table new file mode 100644 index 00000000000..263f0ac6b9d --- /dev/null +++ b/scripts/pi_http/trie-table @@ -0,0 +1,18 @@ + + + trie_table + mysql + ruleidDB_INT + prefixDB_STR + attrsDB_STR + priorityDB_INT + + + + dr_partitions + mysql + idDB_INT + partition_nameDB_STR + db_urlDB_STR + trie_tableDB_STR + diff --git a/scripts/postgres/trie-create.sql b/scripts/postgres/trie-create.sql new file mode 100644 index 00000000000..fd2a8f23bbe --- /dev/null +++ b/scripts/postgres/trie-create.sql @@ -0,0 +1,18 @@ +INSERT INTO version (table_name, table_version) values ('trie_table','1'); +CREATE TABLE trie_table ( + ruleid SERIAL PRIMARY KEY NOT NULL, + prefix VARCHAR(64) NOT NULL, + attrs VARCHAR(255) DEFAULT NULL, + priority INTEGER DEFAULT 1 NOT NULL +); + +ALTER SEQUENCE trie_table_ruleid_seq MAXVALUE 2147483647 CYCLE; +INSERT INTO version (table_name, table_version) values ('dr_partitions','1'); +CREATE TABLE dr_partitions ( + id SERIAL PRIMARY KEY NOT NULL, + partition_name VARCHAR(255) NOT NULL, + db_url VARCHAR(255) NOT NULL, + trie_table VARCHAR(255) +); + +ALTER SEQUENCE dr_partitions_id_seq MAXVALUE 2147483647 CYCLE; diff --git a/scripts/sqlite/trie-create.sql b/scripts/sqlite/trie-create.sql new file mode 100644 index 00000000000..585e7dcbb97 --- /dev/null +++ b/scripts/sqlite/trie-create.sql @@ -0,0 +1,16 @@ +INSERT INTO version (table_name, table_version) values ('trie_table','1'); +CREATE TABLE trie_table ( + ruleid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + prefix CHAR(64) NOT NULL, + attrs CHAR(255) DEFAULT NULL, + priority INTEGER DEFAULT 1 NOT NULL +); + +INSERT INTO version (table_name, table_version) values ('dr_partitions','1'); +CREATE TABLE dr_partitions ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + partition_name CHAR(255) NOT NULL, + db_url CHAR(255) NOT NULL, + trie_table CHAR(255) +); +