From 5cf5d3513aa8c83962bb3ef2aa8fb9de8a0f2bfa Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Wed, 15 Jan 2025 01:44:47 -0500 Subject: [PATCH] minor improvements --- brc | 12 +- brc2 | 10 + brc3 | 6 + .../etc/mysql/mariadb.conf.d/80-iank.cnf | 184 ++++++++++++++++++ fsf-script-lib | 166 ++++++++++++---- sl/.iank/fsf-script-lib | 1 + 6 files changed, 336 insertions(+), 43 deletions(-) create mode 100644 filesystem/etc/mysql/mariadb.conf.d/80-iank.cnf create mode 120000 sl/.iank/fsf-script-lib diff --git a/brc b/brc index b5ed9e1..cbc1b76 100644 --- a/brc +++ b/brc @@ -1380,8 +1380,16 @@ dus() { } ccomp du dus -# echo args +# echo args, one per line ea() { + local arg + for arg; do + printf "%s\n" "$arg" + done +} + +# echo args verbosely +eav() { if (( ! $# )); then echo no args fi @@ -3522,7 +3530,7 @@ m2() { printf "%s\n" "$*" >&2; "$@"; } # echo args, then run args in background, discarding output. mq() { printf "%s\n" "$*"; "$@" &>/dev/null & } -# print args, then prompt to confirm running them. +# m ask. print args, then prompt to confirm running them. ma() { printf "%s\n" "$PWD $ $*" read -rs || return 1 diff --git a/brc2 b/brc2 index ec740c0..db85357 100644 --- a/brc2 +++ b/brc2 @@ -1222,6 +1222,8 @@ beetag() { ;; # down char '[B') + echo ok >>/tmp/x + # skip forward, but show the last few songs anyways. skip_start=0 skip_lookback=3 @@ -3073,6 +3075,14 @@ mns() { # mount namespace } +mnsr() { # mns run (as normal user) + local ns=$1 + shift + env-tmpf "$@" + mns-setup $ns + m sudm -E /usr/bin/nsenter --mount=/root/mount_namespaces/$ns sudo -u iank -i bash -c ". $tmpf" +} + mnsd() { # mount namespace + systemd network namespace local ns unit user tmpf pre_check pid alt_user=false local -a final_args diff --git a/brc3 b/brc3 index 3b75081..82ea3db 100644 --- a/brc3 +++ b/brc3 @@ -119,3 +119,9 @@ schroot-cleanup() { # misc bash tip. If we have a set of lines we want to know if one is a specific string, we can use =~ with: # regex="("$'\n'"|^)$string("$'\n'"|$)" + + +# usage: pass encoded string on stdin +url-decode() { + python3 -c "import sys; from urllib.parse import unquote; print(unquote(sys.stdin.read()));" + } diff --git a/filesystem/etc/mysql/mariadb.conf.d/80-iank.cnf b/filesystem/etc/mysql/mariadb.conf.d/80-iank.cnf new file mode 100644 index 0000000..04bcfd4 --- /dev/null +++ b/filesystem/etc/mysql/mariadb.conf.d/80-iank.cnf @@ -0,0 +1,184 @@ +# iank: i changed a few things. source: https://gist.github.com/fevangelou/0da9941e67a9c9bb2596 +# +# === Optimized my.cnf configuration for MySQL/MariaDB (on Ubuntu, CentOS, Almalinux etc. servers) === +# +# by Fotis Evangelou, developer of Engintron (engintron.com) +# +# ~ Updated September 2024 ~ +# +# +# The settings provided below are a starting point for a 8-16 GB RAM server with 4-8 CPU cores. +# If you have different resources available you should adjust accordingly to save CPU, RAM & disk I/O usage. +# +# The settings marked with a specific comment or the word "UPD" (after the value) +# should be adjusted for your system by using database diagnostics tools like: +# +# https://github.com/major/MySQLTuner-perl +# or +# https://github.com/BMDan/tuning-primer.sh +# +# Run either of these scripts before optimizing your database, at least 1 hr after the optimization & finally +# at least once a day for 3 days (without restarting the database) to see how your server performs and if you need +# to re-adjust anything. The more MySQL/MariaDB runs without restarting, the more usage data it gathers, so these +# diagnostics scripts will report in mode detail how MySQL/MariaDB performs. +# +# +# IMPORTANT NOTE: If there is NO comment after a setting value, then 99,9% of the times you won't need to adjust it. +# +# +# --- THINGS TO DO AFTER YOU UPDATE MY.CNF - TROUBLESHOOTING --- +# +# If any terminal commands are mentioned, make sure you execute them as "root" user. +# +# If MySQL or MariaDB cannot start (or restart), then perform the following actions. +# +# 1. If the server had the stock database configuration and you added or updated any +# "innodb_log_*" settings (as suggested below), then execute these commands ONLY +# the first time you apply this configuration: +# +# $ chown -R mysql:mysql /var/lib/mysql +# $ service mysql restart +# +# or use the shorthand command: +# $ chown -R mysql:mysql /var/lib/mysql; service mysql restart +# +# 2. If the setting "bind-address" is not commented out, then make sure the file /etc/hosts is +# properly configured. A good example of a "clean" /etc/hosts file is something like this: +# +# 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +# ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +# 1.2.3.4 hostname.domain.tld hostname # <-- Replace accordingly! +# +# Finally restart the database service: +# +# $ service mysql restart +# +# 3. If the database service cannot restart even after the first 2 steps, make sure the database data folder +# (common for either MySQL or MariaDB) "/var/lib/mysql" is owned by the "mysql" user AND group. +# Additionally, the folder itself can have 0751 or 0755 file permissions. To fix it, simply do this: +# $ chown -R mysql:mysql /var/lib/mysql +# $ chmod 0755 /var/lib/mysql +# +# Finally restart the database service: +# +# $ service mysql restart +# +# +# ~ FIN ~ + + +[mysqld] +max_allowed_packet = 256M +max_connect_errors = 1000000 +skip_external_locking +skip_name_resolve + +# === SQL Compatibility Mode === +# Enable for b/c with databases created in older MySQL/MariaDB versions +# (e.g. when using null dates) +#sql_mode = ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES +# For maximum compatibility, just use: +#sql_mode = "" + +# === InnoDB Settings === +default_storage_engine = InnoDB +innodb_buffer_pool_instances = 4 # Use 1 instance per 1GB of InnoDB pool size - max is 64 +innodb_buffer_pool_size = 4G # Use up to 70-80% of RAM +innodb_file_per_table = 1 +innodb_flush_log_at_trx_commit = 0 +innodb_flush_method = O_DIRECT +innodb_log_buffer_size = 16M +innodb_log_file_size = 1G +innodb_sort_buffer_size = 4M # UPD - Defines how much data is read into memory for sorting operations before writing to disk (default is 1M / max is 64M) +innodb_stats_on_metadata = 0 +#innodb_use_fdatasync = 1 # Only (!) for MySQL v8.0.26+ + +#innodb_temp_data_file_path = ibtmp1:64M:autoextend:max:20G # Control the maximum size for the ibtmp1 file +#innodb_thread_concurrency = 4 # Optional: Set to the number of CPUs on your system (minus 1 or 2) to better + # contain CPU usage. E.g. if your system has 8 CPUs, try 6 or 7 and check + # the overall load produced by MySQL/MariaDB. +innodb_read_io_threads = 64 +innodb_write_io_threads = 64 +innodb_io_capacity = 2000 # Depends on the storage tech - use 2000 for SSD, more for NVMe +#innodb_io_capacity_max = 4000 # Usually double the value of innodb_io_capacity + +# === MyISAM Settings === +# The following 3 options are ONLY supported by MariaDB & up to MySQL 5.7 +# Do NOT un-comment on MySQL 8.x+ +#query_cache_limit = 4M # UPD +#query_cache_size = 64M # UPD +#query_cache_type = 1 # Enabled by default + +key_buffer_size = 16M # UPD + +low_priority_updates = 1 +concurrent_insert = 2 + +# === Connection Settings === +max_connections = 90 # UPD - Important: high no. of connections = high RAM consumption + +back_log = 512 +thread_cache_size = 100 +thread_stack = 192K + +interactive_timeout = 180 +wait_timeout = 180 + +# For MySQL 5.7+ only (disabled by default) +#max_execution_time = 90000 # Set a timeout limit for SELECT statements (value in milliseconds). + # This option may be useful to address aggressive crawling on large sites, + # but it can also cause issues (e.g. with backups). So use with extreme caution and test! + # More info at: https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_execution_time + +# For MariaDB 10.1.1+ only (disabled by default) +#max_statement_time = 90 # The equivalent of "max_execution_time" in MySQL 5.7+ (set above) + # The variable is of type double, thus you can use subsecond timeout. + # For example you can use value 0.01 for 10 milliseconds timeout. + # More info at: https://mariadb.com/kb/en/aborting-statements/ + +# === Buffer Settings === +# Handy tip for managing your database's RAM usage: +# The following values should be treated carefully as they are added together and then multiplied by your "max_connections" value. +# Other options will also add up to RAM consumption (e.g. tmp_table_size). So don't go switching your "join_buffer_size" to 1G, it's harmful & inefficient. +# Use one of the database diagnostics tools mentioned at the top of this file to count your database's potential total RAM usage, so you know if you are within +# reasonable limits. Remember that other services will require enough RAM to operate properly (like Apache or PHP-FPM), so set your limits wisely. +join_buffer_size = 4M # UPD +read_buffer_size = 3M # UPD +read_rnd_buffer_size = 4M # UPD +sort_buffer_size = 4M # UPD + +# === Table Settings === +# In systemd managed systems like Ubuntu 16.04+ or CentOS 7+, you need to perform an extra action for table_open_cache & open_files_limit +# to be overriden (also see comment next to open_files_limit). +# E.g. for MySQL 5.7, please check: https://dev.mysql.com/doc/refman/5.7/en/using-systemd.html +# and for MariaDB check: https://mariadb.com/kb/en/library/systemd/ +table_definition_cache = 50000 # UPD +table_open_cache = 50000 # UPD +open_files_limit = 60000 # UPD - This can be 2x to 3x the table_open_cache value or match the system's + # open files limit usually set in /etc/sysctl.conf and /etc/security/limits.conf + # In systemd managed systems this limit must also be set in: + # - /etc/systemd/system/mysql.service.d/override.conf (for MySQL 5.7+ in Ubuntu) or + # - /etc/systemd/system/mysqld.service.d/override.conf (for MySQL 5.7+ in CentOS) or + # - /etc/systemd/system/mariadb.service.d/override.conf (for MariaDB) + # otherwise changing open_files_limit will have no effect. + # + # To edit the right file execute: + # $ systemctl edit mysql (or mysqld or mariadb) + # and set "LimitNOFILE=" to something like 100000 or more (depending on your system limits for MySQL) + # or use "LimitNOFILE=infinity" for MariaDB only. + # Finally merge the changes with: + # $ systemctl daemon-reload; systemctl restart mysql (or mysqld or mariadb) + +max_heap_table_size = 128M # Increase to 256M, 512M or 1G if you have lots of temporary tables because of missing indices in JOINs +tmp_table_size = 128M # Use same value as max_heap_table_size + +# === Search Settings === +ft_min_word_len = 3 # Minimum length of words to be indexed for search results + +# === Binary Logging === +disable_log_bin = 1 # Binary logging disabled by default +#log_bin # To enable binary logging, uncomment this line & only one of the following 2 lines + # that corresponds to your actual MySQL/MariaDB version. + # Remember to comment out the line with "disable_log_bin". +#expire_logs_days = 1 # Keep logs for 1 day - For MySQL 5.x & MariaDB before 10.6 only +#binlog_expire_logs_seconds = 86400 # Keep logs for 1 day (in seconds) - For MySQL 8+ & MariaDB 10.6+ only diff --git a/fsf-script-lib b/fsf-script-lib index 950d612..6392b74 100644 --- a/fsf-script-lib +++ b/fsf-script-lib @@ -42,15 +42,24 @@ set +e # low pri todo: compare this to a screen or tmux log. slog() { local log_tmp log_dir log_path - log_tmp=(~/log) - log_dir="${log_tmp[0]}" - mkdir -p "$log_dir" - - log_path="$log_dir/$(date +%F_%H_%M_%S).log" - if [[ -e $log_path ]]; then - sleep 1 + case $1 in + -l) + log_path="$2" + shift 2 + ;; + esac + + if [[ ! $log_path ]]; then + log_tmp=(~/log) + log_dir="${log_tmp[0]}" + mkdir -p "$log_dir" log_path="$log_dir/$(date +%F_%H_%M_%S).log" + if [[ -e $log_path ]]; then + sleep 1 + log_path="$log_dir/$(date +%F_%H_%M_%S).log" + fi fi + script -t $log_path.s 2> $log_path.t } @@ -58,12 +67,33 @@ slog() { # usage: m COMMAND... # # echo COMMAND then run it. -m() { printf "+ %s\n" "$*"; "$@"; } +m() { printf "+ %s\n" "$*" >&2; "$@"; } + +# mb, maybe. echo args if they fail. +mb() { + local ret=0 + "$@" || ret=$? + if (( ret )); then + printf "error: exit code $ret from: %s\n" "$*" >&2 + fi + return $ret +} # usage: e [MESSAGE...] # # echo MESSAGE. -e() { printf "${0##*/}: %s\n" "$*"; } +if [[ $- == *i* ]]; then + # prefix doesn't make sense in interactive shell + e() { printf "%s\n" "$*"; } +else + e() { + if [[ $debug_interact ]]; then + printf "%s\n" "$*" + else + printf "${0##*/}: %s\n" "$*" + fi + } +fi # usage: err [ERROR_MESSAGE...] # @@ -109,17 +139,24 @@ debug-setx() { } # internal function called via debug-setx trap. _debug-setx-func() { - local doprint=false func_depth random_delimiter func + trap DEBUG + local doprint=false random_delimiter func cmd col debug_cmd if [[ $debug_max_stack_depth ]]; then - func_depth=$(( debug_max_stack_depth + 2 )) + funcname_count_max=$(( debug_max_stack_depth + 1 )) else - # at a script top level we see depth of 2: debug-setx-func main - func_depth=2 + # within 0 func level, FUNCNAME has 2 in here, eg debug-setx-func main. + funcname_count_max=2 + fi + + if [[ $BASH_SUBSHELL != [01] ]]; then + trap _debug-setx-func DEBUG + return 0 fi if $debug_skip; then debug_skip=false + trap _debug-setx-func DEBUG return 0 fi @@ -145,17 +182,19 @@ _debug-setx-func() { fi ## block for default command & function ignoring - if ! $doprint && (( ${#FUNCNAME[*]} <= func_depth )); then + if ! $doprint && (( ${#FUNCNAME[*]} <= funcname_count_max )); then + # debug + #e "${FUNCNAME[*]} ${#FUNCNAME[*]} <= $funcname_count_max" case $BASH_COMMAND in - e\ *|err\ *|echo\ *|printf\ *|read\ *) + e\ *|err\ *|echo\ *|printf\ *|read\ *|h|ux|ux\ *) : ;; for\ *) # try to print just the first for loop iteration. - _script_pp_cur_for="$BASH_COMMAND ${BASH_LINENO[0]} ${FUNCNAME[1]}" - if [[ $_script_pp_cur_for != "$_script_pp_last_for" ]]; then + _debug_setx_cur_for="$BASH_COMMAND ${BASH_LINENO[0]} ${FUNCNAME[1]}" + if [[ $_debug_setx_cur_for != "$_debug_setx_last_for" ]]; then doprint=true fi - _script_pp_last_for="_script_pp_cur_for" + _debug_setx_last_for="$_debug_setx_cur_for" ;; *) doprint=true @@ -166,64 +205,108 @@ _debug-setx-func() { fi if $doprint; then - if [[ $BASH_COMMAND != *\`* && $BASH_COMMAND != *\$\(* && $BASH_COMMAND == *\$* ]]; then + # Expanding $1 will be empty, so skip expansion in that case, and some others. + # + # TODO: we should be able to grab $1 from the function tracing facilities. + # + # Note, we could split the command up and expand other parts + # but I'm not going to bother. + if [[ $BASH_COMMAND != *\$[0-9]* && $BASH_COMMAND != *\`* && $BASH_COMMAND != *\$\(* && $BASH_COMMAND == *\$* ]]; then random_delimiter=$RANDOM$RANDOM$RANDOM # shellcheck disable=SC1003 _d="$BASH_COMMAND" _d="${_d//'\\'/SUBfopguensOfRifejmu${random_delimiter}SUB}" _d="${_d//\\[adehHjlnrstT@AuvVwW\!\#\$[\]]/\\&}" _d="${_d//\\D\{*\}/\\&}" _d="${_d//\\[012][0-9][0-9]/\\&}" _d="${_d//\\3[0-6][0-9]/\\&}" _d="${_d//\\37[0-7]/\\&}" _d="${_d//SUBfopguensOfRifejmu${random_delimiter}SUB/'\\\\'}" - echo "+ ${_d@P}" + cmd="${_d@P}" else - echo "+ $BASH_COMMAND" + cmd="$BASH_COMMAND" fi + if [[ $COLUMNS == *[0-9]* ]]; then + # generally fill up 2 rows + col=$(( COLUMNS * 2 - 20 )) + else + col=150 + fi + if (( ${#cmd} > col )); then + cmd="$(( ${#cmd} - col)) cut: ${cmd:0:col}" + fi + echo "+ $cmd" >&2 fi if $debug_prompt; then - read -rs + # we can type in commands here to inspect the program state. + read -r -e -p "DEBUG $ " debug_cmd + while [[ $debug_cmd ]]; do + eval $debug_cmd + read -r -e -p "DEBUG $ " debug_cmd + done debug_prompt=false fi + trap _debug-setx-func DEBUG } # usage: ux [COMMAND...] # -# disable/unset debug-setx default command printing. +# disable/unset debug-setx default command printing. Runs COMMAND if +# given. Command arg exists only because the args would be otherwise +# unused. ux() { debug_print_default=false - "$@" + if [[ $1 ]]; then "$@"; fi +} + +# usage: h [COMMAND...] +# +# Print the next command then wait for user to press enter (or ctrl-c to +# exit). +# +# note: h = random free letter. +h() { + if [[ $1 ]]; then + debug_prompt=true + "$@" + else + debug_prompt=true + fi } # usage: u1 [COMMAND...] # # Disable debug-setx printing just for the next command. u1() { - debug_skip=true - "$@" + # we have this strange logic because [[ is a command, easier to not + # have that be between debug_skip=true and $@ and trigger a debug + # trap. + if [[ $1 ]]; then + debug_skip=true + "$@" + else + debug_skip=true + fi } # usage: dx [COMMAND...] # # Undo ux() and reenable debug-setx printing. dx() { - debug_print_default=true - "$@" + if [[ $1 ]]; then + debug_print_default=true + "$@" + else + debug_print_default=true + fi } # usage: d1 [COMMAND...] # # For just one command, reenable debug-setx printing. d1() { - debug_force_print=true - "$@" + if [[ $1 ]]; then + debug_force_print=true + "$@" + else + debug_force_print=true + fi } -# usage: h [COMMAND...] -# -# Print the next command then wait for user to press enter (or ctrl-c to -# exit). -# -# note: h = random free letter. -h() { - debug_prompt=true - "$@" -} # usage: setx-func [FUNCTION...] # @@ -236,7 +319,8 @@ setx-func() { debug_print_funcs=() fi - if (( $# )); then + e "${FUNCNAME[*]}" + if [[ $1 ]]; then new_funcs="$*" else new_funcs="${FUNCNAME[0]}" diff --git a/sl/.iank/fsf-script-lib b/sl/.iank/fsf-script-lib new file mode 120000 index 0000000..62b8373 --- /dev/null +++ b/sl/.iank/fsf-script-lib @@ -0,0 +1 @@ +../../fsf-script-lib \ No newline at end of file -- 2.30.2