# Truenas Maintenance Logs (10-1-2025) ### What started it all # Stats listing script: ```bash cat >/tmp/nc_sysdump.sh <<'EOF' #!/usr/bin/env bash set -euo pipefail TIMEOUT="${TIMEOUT:-5}" # seconds hr(){ printf '%*s\n' "${COLUMNS:-88}" '' | tr ' ' '-'; } ts(){ date +"%Y-%m-%d %H:%M:%S %Z"; } safe(){ timeout "${TIMEOUT}" bash -lc "$*" 2>/dev/null || echo "[timeout] $*"; } echo "NEXTCLOUD PERF SNAPSHOT ($(ts))" hr echo "[OS]" safe 'uname -a' echo echo "[CPU]" safe "lscpu | egrep 'Model name|CPU\\(s\\)|Thread|Core|Socket|MHz|Cache'" echo echo "[Memory]" safe 'free -h' echo echo "[Block Devices]" safe 'lsblk -o NAME,SIZE,TYPE,ROTA,MODEL,MOUNTPOINT | sed "s/^/ /"' echo echo "[ZFS Pools]" safe 'zpool list -v' echo echo "[ZFS Pool Status (incl. SLOG/L2ARC)]" safe 'zpool status -v' echo echo "[ARC]" if command -v arcstat >/dev/null 2>&1; then safe 'arcstat 1 1' else awk ' $1=="size" {size=$3} $1=="c" {c=$3} $1=="c_max" {cmax=$3} END { printf "size: %s bytes\nc: %s bytes\nc_max: %s bytes\n", size, c, cmax } ' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo "arc stats not available" fi echo echo "[Likely datasets for Nextcloud/Postgres/Redis]" safe "zfs list -o name,used,refer,mountpoint,recordsize,compression,primarycache,logbias | egrep -i 'nextcloud|appdata|postgres|pgdata|pg[_-]wal|redis|docker|k3s|ix-applications' || true" echo echo "[ZFS properties for pgdata/pg_wal (if found)]" for ds in $(zfs list -H -o name | egrep -i 'pgdata|pg[_-]wal' || true); do echo "Dataset: $ds" safe "zfs get -H recordsize,atime,compression,logbias,primarycache '$ds'" done echo echo "[App Platform]" echo "- TrueNAS SCALE Apps (k3s):" if systemctl is-active --quiet k3s 2>/dev/null; then echo "k3s: active" if command -v kubectl >/dev/null 2>&1; then safe "kubectl get pods -A | egrep -i 'nextcloud|postgres|redis' || true" else echo "kubectl not found" fi else echo "k3s: inactive" fi echo echo "- Docker:" if command -v docker >/dev/null 2>&1; then safe "docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}' | egrep -i 'nextcloud|postgres|redis' || true" echo echo "[Docker volumes and mounts (top 20)]" safe "docker ps --no-trunc --format '{{.Names}} {{.Mounts}}' | head -20" else echo "docker not found" fi echo echo "[Nextcloud hints]" # try to locate a Nextcloud config quickly (limited depth) safe "find / -maxdepth 4 -type f -name config.php 2>/dev/null | egrep -i 'nextcloud' | head -3" echo echo "[Network basics]" safe "ip -brief addr" safe "ss -ltpn | egrep -i 'postgres|redis|:80 |:443 ' || true" echo echo "[Kernel tunables impacting Redis/DB]" safe "sysctl vm.overcommit_memory" if [ -r /sys/kernel/mm/transparent_hugepage/enabled ]; then echo -n "THP: "; cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || true fi safe "sysctl vm.swappiness" echo echo "[Cron/Timers]" safe "systemctl is-active cron || systemctl is-active crond || systemctl is-active cron.service" safe "systemctl list-timers --all | head -15" echo echo "[Postgres quick info]" if command -v psql >/dev/null 2>&1; then safe "psql --version" else echo "psql not found" fi echo echo "[Redis quick info]" if command -v redis-cli >/dev/null 2>&1; then safe "redis-cli INFO | egrep 'redis_version|mem_total|maxmemory|appendonly' || true" else echo "redis-cli not found" fi echo echo "DONE. Copy everything above and paste it here." EOF sudo bash /tmp/nc_sysdump.sh ``` # NEXT STEPS ```bash admin@truenas[~]$ sudo zpool remove Pool2 d72d8e8a-c905-42ac-9842-1b54d6a6e62c~ cannot remove d72d8e8a-c905-42ac-9842-1b54d6a6e62c~: no such device in pool admin@truenas[~]$ sudo zpool remove Pool2 d72d8e8a-c905-42ac-9842-1b54d6a6e62c admin@truenas[~]$ zpool status Pool2 zsh: command not found: zpool admin@truenas[~]$ sudo zpool status Pool2 pool: Pool2 state: ONLINE scan: scrub repaired 0B in 11:09:30 with 0 errors on Sun Sep 14 11:09:33 2025 config: NAME STATE READ WRITE CKSUM Pool2 ONLINE 0 0 0 raidz2-0 ONLINE 0 0 0 77f4ab18-6075-4e20-939a-c6c8022f01e3 ONLINE 0 0 0 0636126f-88e7-41db-a45c-3ce63eb4cdf9 ONLINE 0 0 0 a9532cf0-8ad7-4100-9597-21475bcc379a ONLINE 0 0 0 230dd4b9-a882-43b9-8767-a380118cd644 ONLINE 0 0 0 6a34f32a-9644-4b87-a1d0-d81d25719324 ONLINE 0 0 0 ac8e92c0-d113-4b21-9903-b02baa6e79f2 ONLINE 0 0 0 79d6a2b8-da07-495d-b5cb-53a2861a3cea ONLINE 0 0 0 c81e032d-12a8-4958-bdbc-e879fdaaf5d2 ONLINE 0 0 0 6a6f7267-93aa-4d40-8ebb-b4b3e636a0c7 ONLINE 0 0 0 1f68b208-eb04-4f81-9325-9e49132586fc ONLINE 0 0 0 spares 74246ece-8954-43e8-9e7b-1835d394487c AVAIL errors: No known data errors admin@truenas[~]$ PGDATA="Pool2/ix-applications/releases/nextcloud/volumes/ix_volumes/pgData" admin@truenas[~]$ sudo zfs set recordsize=8K atime=off compression=lz4 logbias=latency primarycache=all "$PGDATA" admin@truenas[~]$ echo never | sudo tee /sys/kernel/mm/transparent_hugepage/enabled >/dev/null admin@truenas[~]$ echo never | sudo tee /sys/kernel/mm/transparent_hugepage/defrag >/dev/null admin@truenas[~]$ sudo tee /etc/systemd/system/disable-thp.service >/dev/null <<'EOF' heredoc> [Unit] Description=Disable Transparent Huge Pages After=multi-user.target [Service] Type=oneshot ExecStart=/bin/sh -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' ExecStart=/bin/sh -c 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' RemainAfterExit=yes [Install] WantedBy=multi-user.target EOF sudo systemctl daemon-reload sudo systemctl enable --now disable-thp.service Created symlink /etc/systemd/system/multi-user.target.wants/disable-thp.service → /etc/systemd/system/disable-thp.service. admin@truenas[~]$ sudo sysctl vm.swappiness=1 vm.swappiness = 1 admin@truenas[~]$ echo 'vm.swappiness=1' | sudo tee /etc/sysctl.d/99-redis-db.conf >/dev/null admin@truenas[~]$ sudo sysctl --system * Applying /etc/sysctl.d/10-truenas.conf ... * Applying /usr/lib/sysctl.d/50-coredump.conf ... * Applying /usr/lib/sysctl.d/50-pid-max.conf ... * Applying /usr/lib/sysctl.d/99-protect-links.conf ... * Applying /etc/sysctl.d/99-redis-db.conf ... * Applying /etc/sysctl.d/99-sysctl.conf ... * Applying /etc/sysctl.conf ... kernel.panic = 10 kernel.panic_on_oops = 1 kernel.panic_on_io_nmi = 1 kernel.panic_on_unrecovered_nmi = 1 kernel.unknown_nmi_panic = 1 kernel.watchdog_thresh = 60 kernel.core_pattern = |/lib/systemd/systemd-coredump %P %u %g %s %t 9223372036854775808 %h kernel.core_pipe_limit = 16 fs.suid_dumpable = 2 kernel.pid_max = 4194304 fs.protected_fifos = 1 fs.protected_hardlinks = 1 fs.protected_regular = 2 fs.protected_symlinks = 1 vm.swappiness = 1 admin@truenas[~]$ cd /mnt/Pool2/ix-applications/ admin@truenas[/mnt/Pool2/ix-applications]$ ls app_migrations.json backups catalogs config.json default_volumes k3s migrations.json releases admin@truenas[/mnt/Pool2/ix-applications]$ cd releases/nextcloud/volumes/ix_volumes admin@truenas[...releases/nextcloud/volumes/ix_volumes]$ ls data html pgBackup pgData admin@truenas[...releases/nextcloud/volumes/ix_volumes]$ cd html admin@truenas[...ses/nextcloud/volumes/ix_volumes/html]$ ls 3rdparty COPYING composer.json config core custom_apps dist index.php nextcloud-init-sync.lock ocs package-lock.json public.php resources status.php version.php AUTHORS apps composer.lock console.php cron.php data index.html lib occ ocs-provider package.json remote.php robots.txt themes admin@truenas[...ses/nextcloud/volumes/ix_volumes/html]$ cd .. admin@truenas[...releases/nextcloud/volumes/ix_volumes]$ cd .. admin@truenas[...plications/releases/nextcloud/volumes]$ ls ix_volumes admin@truenas[...plications/releases/nextcloud/volumes]$ cd ix_volumes admin@truenas[...releases/nextcloud/volumes/ix_volumes]$ ls data html pgBackup pgData admin@truenas[...releases/nextcloud/volumes/ix_volumes]$ cd pgData cd: permission denied: pgData admin@truenas[...releases/nextcloud/volumes/ix_volumes]$ sudo su root@truenas[...releases/nextcloud/volumes/ix_volumes]# cd pgData root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# ls PG_VERSION base global pg_commit_ts pg_dynshmem pg_hba.conf pg_ident.conf pg_logical pg_multixact pg_notify pg_replslot pg_serial pg_snapshots pg_stat pg_stat_tmp pg_subtrans pg_tblspc pg_twophase pg_wal pg_xact postgresql.auto.conf postgresql.conf postmaster.opts postmaster.pid root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# cat postgresql.conf # ----------------------------- # PostgreSQL configuration file # ----------------------------- # # This file consists of lines of the form: # # name = value # # (The "=" is optional.) Whitespace may be used. Comments are introduced with # "#" anywhere on a line. The complete list of parameter names and allowed # values can be found in the PostgreSQL documentation. # # The commented-out settings shown in this file represent the default values. # Re-commenting a setting is NOT sufficient to revert it to the default value; # you need to reload the server. # # This file is read on server startup and when the server receives a SIGHUP # signal. If you edit the file on a running system, you have to SIGHUP the # server for the changes to take effect, run "pg_ctl reload", or execute # "SELECT pg_reload_conf()". Some parameters, which are marked below, # require a server shutdown and restart to take effect. # # Any parameter can also be given as a command-line option to the server, e.g., # "postgres -c log_connections=on". Some parameters can be changed at run time # with the "SET" SQL command. # # Memory units: kB = kilobytes Time units: ms = milliseconds # MB = megabytes s = seconds # GB = gigabytes min = minutes # TB = terabytes h = hours # d = days #------------------------------------------------------------------------------ # FILE LOCATIONS #------------------------------------------------------------------------------ # The default values of these variables are driven from the -D command-line # option or PGDATA environment variable, represented here as ConfigDir. #data_directory = 'ConfigDir' # use data in another directory # (change requires restart) #hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file # (change requires restart) #ident_file = 'ConfigDir/pg_ident.conf' # ident configuration file # (change requires restart) # If external_pid_file is not explicitly set, no extra PID file is written. #external_pid_file = '' # write an extra PID file # (change requires restart) #------------------------------------------------------------------------------ # CONNECTIONS AND AUTHENTICATION #------------------------------------------------------------------------------ # - Connection Settings - listen_addresses = '*' # comma-separated list of addresses; # defaults to 'localhost'; use '*' for all # (change requires restart) #port = 5432 # (change requires restart) max_connections = 100 # (change requires restart) #superuser_reserved_connections = 3 # (change requires restart) #unix_socket_directories = '/var/run/postgresql' # comma-separated list of directories # (change requires restart) #unix_socket_group = '' # (change requires restart) #unix_socket_permissions = 0777 # begin with 0 to use octal notation # (change requires restart) #bonjour = off # advertise server via Bonjour # (change requires restart) #bonjour_name = '' # defaults to the computer name # (change requires restart) # - TCP settings - # see "man tcp" for details #tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; # 0 selects the system default #tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; # 0 selects the system default #tcp_keepalives_count = 0 # TCP_KEEPCNT; # 0 selects the system default #tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; # 0 selects the system default # - Authentication - #authentication_timeout = 1min # 1s-600s #password_encryption = md5 # md5 or scram-sha-256 #db_user_namespace = off # GSSAPI using Kerberos #krb_server_keyfile = '' #krb_caseins_users = off # - SSL - #ssl = off #ssl_ca_file = '' #ssl_cert_file = 'server.crt' #ssl_crl_file = '' #ssl_key_file = 'server.key' #ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers #ssl_prefer_server_ciphers = on #ssl_ecdh_curve = 'prime256v1' #ssl_min_protocol_version = 'TLSv1.2' #ssl_max_protocol_version = '' #ssl_dh_params_file = '' #ssl_passphrase_command = '' #ssl_passphrase_command_supports_reload = off #------------------------------------------------------------------------------ # RESOURCE USAGE (except WAL) #------------------------------------------------------------------------------ # - Memory - shared_buffers = 128MB # min 128kB # (change requires restart) #huge_pages = try # on, off, or try # (change requires restart) #temp_buffers = 8MB # min 800kB #max_prepared_transactions = 0 # zero disables the feature # (change requires restart) # Caution: it is not advisable to set max_prepared_transactions nonzero unless # you actively intend to use prepared transactions. #work_mem = 4MB # min 64kB #hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem #maintenance_work_mem = 64MB # min 1MB #autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem #logical_decoding_work_mem = 64MB # min 64kB #max_stack_depth = 2MB # min 100kB #shared_memory_type = mmap # the default is the first option # supported by the operating system: # mmap # sysv # windows # (change requires restart) dynamic_shared_memory_type = posix # the default is the first option # supported by the operating system: # posix # sysv # windows # mmap # (change requires restart) # - Disk - #temp_file_limit = -1 # limits per-process temp file space # in kilobytes, or -1 for no limit # - Kernel Resources - #max_files_per_process = 1000 # min 64 # (change requires restart) # - Cost-Based Vacuum Delay - #vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) #vacuum_cost_page_hit = 1 # 0-10000 credits #vacuum_cost_page_miss = 10 # 0-10000 credits #vacuum_cost_page_dirty = 20 # 0-10000 credits #vacuum_cost_limit = 200 # 1-10000 credits # - Background Writer - #bgwriter_delay = 200ms # 10-10000ms between rounds #bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables #bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round #bgwriter_flush_after = 512kB # measured in pages, 0 disables # - Asynchronous Behavior - #effective_io_concurrency = 1 # 1-1000; 0 disables prefetching #maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching #max_worker_processes = 8 # (change requires restart) #max_parallel_maintenance_workers = 2 # taken from max_parallel_workers #max_parallel_workers_per_gather = 2 # taken from max_parallel_workers #parallel_leader_participation = on #max_parallel_workers = 8 # maximum number of max_worker_processes that # can be used in parallel operations #old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate # (change requires restart) #backend_flush_after = 0 # measured in pages, 0 disables #------------------------------------------------------------------------------ # WRITE-AHEAD LOG #------------------------------------------------------------------------------ # - Settings - #wal_level = replica # minimal, replica, or logical # (change requires restart) #fsync = on # flush data to disk for crash safety # (turning this off can cause # unrecoverable data corruption) #synchronous_commit = on # synchronization level; # off, local, remote_write, remote_apply, or on #wal_sync_method = fsync # the default is the first option # supported by the operating system: # open_datasync # fdatasync (default on Linux) # fsync # fsync_writethrough # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers # (change requires restart) #wal_writer_delay = 200ms # 1-10000 milliseconds #wal_writer_flush_after = 1MB # measured in pages, 0 disables #wal_skip_threshold = 2MB #commit_delay = 0 # range 0-100000, in microseconds #commit_siblings = 5 # range 1-1000 # - Checkpoints - #checkpoint_timeout = 5min # range 30s-1d max_wal_size = 1GB min_wal_size = 80MB #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 #checkpoint_flush_after = 256kB # measured in pages, 0 disables #checkpoint_warning = 30s # 0 disables # - Archiving - #archive_mode = off # enables archiving; off, on, or always # (change requires restart) #archive_command = '' # command to use to archive a logfile segment # placeholders: %p = path of file to archive # %f = file name only # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' #archive_timeout = 0 # force a logfile segment switch after this # number of seconds; 0 disables # - Archive Recovery - # These are only used in recovery mode. #restore_command = '' # command to use to restore an archived logfile segment # placeholders: %p = path of file to restore # %f = file name only # e.g. 'cp /mnt/server/archivedir/%f %p' # (change requires restart) #archive_cleanup_command = '' # command to execute at every restartpoint #recovery_end_command = '' # command to execute at completion of recovery # - Recovery Target - # Set these only when performing a targeted recovery. #recovery_target = '' # 'immediate' to end recovery as soon as a # consistent state is reached # (change requires restart) #recovery_target_name = '' # the named restore point to which recovery will proceed # (change requires restart) #recovery_target_time = '' # the time stamp up to which recovery will proceed # (change requires restart) #recovery_target_xid = '' # the transaction ID up to which recovery will proceed # (change requires restart) #recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed # (change requires restart) #recovery_target_inclusive = on # Specifies whether to stop: # just after the specified recovery target (on) # just before the recovery target (off) # (change requires restart) #recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID # (change requires restart) #recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' # (change requires restart) #------------------------------------------------------------------------------ # REPLICATION #------------------------------------------------------------------------------ # - Sending Servers - # Set these on the master and on any standby that will send replication data. #max_wal_senders = 10 # max number of walsender processes # (change requires restart) #wal_keep_size = 0 # in megabytes; 0 disables #max_slot_wal_keep_size = -1 # in megabytes; -1 disables #wal_sender_timeout = 60s # in milliseconds; 0 disables #max_replication_slots = 10 # max number of replication slots # (change requires restart) #track_commit_timestamp = off # collect timestamp of transaction commit # (change requires restart) # - Master Server - # These settings are ignored on a standby server. #synchronous_standby_names = '' # standby servers that provide sync rep # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name # from standby(s); '*' = all #vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed # - Standby Servers - # These settings are ignored on a master server. #primary_conninfo = '' # connection string to sending server #primary_slot_name = '' # replication slot on sending server #promote_trigger_file = '' # file name whose presence ends recovery #hot_standby = on # "off" disallows queries during recovery # (change requires restart) #max_standby_archive_delay = 30s # max delay before canceling queries # when reading WAL from archive; # -1 allows indefinite delay #max_standby_streaming_delay = 30s # max delay before canceling queries # when reading streaming WAL; # -1 allows indefinite delay #wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name # is not set #wal_receiver_status_interval = 10s # send replies at least this often # 0 disables #hot_standby_feedback = off # send info from standby to prevent # query conflicts #wal_receiver_timeout = 60s # time that receiver waits for # communication from master # in milliseconds; 0 disables #wal_retrieve_retry_interval = 5s # time to wait before retrying to # retrieve WAL after a failed attempt #recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery # - Subscribers - # These settings are ignored on a publisher. #max_logical_replication_workers = 4 # taken from max_worker_processes # (change requires restart) #max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers #------------------------------------------------------------------------------ # QUERY TUNING #------------------------------------------------------------------------------ # - Planner Method Configuration - #enable_bitmapscan = on #enable_hashagg = on #enable_hashjoin = on #enable_indexscan = on #enable_indexonlyscan = on #enable_material = on #enable_mergejoin = on #enable_nestloop = on #enable_parallel_append = on #enable_seqscan = on #enable_sort = on #enable_incremental_sort = on #enable_tidscan = on #enable_partitionwise_join = off #enable_partitionwise_aggregate = off #enable_parallel_hash = on #enable_partition_pruning = on # - Planner Cost Constants - #seq_page_cost = 1.0 # measured on an arbitrary scale #random_page_cost = 4.0 # same scale as above #cpu_tuple_cost = 0.01 # same scale as above #cpu_index_tuple_cost = 0.005 # same scale as above #cpu_operator_cost = 0.0025 # same scale as above #parallel_tuple_cost = 0.1 # same scale as above #parallel_setup_cost = 1000.0 # same scale as above #jit_above_cost = 100000 # perform JIT compilation if available # and query more expensive than this; # -1 disables #jit_inline_above_cost = 500000 # inline small functions if query is # more expensive than this; -1 disables #jit_optimize_above_cost = 500000 # use expensive JIT optimizations if # query is more expensive than this; # -1 disables #min_parallel_table_scan_size = 8MB #min_parallel_index_scan_size = 512kB #effective_cache_size = 4GB # - Genetic Query Optimizer - #geqo = on #geqo_threshold = 12 #geqo_effort = 5 # range 1-10 #geqo_pool_size = 0 # selects default based on effort #geqo_generations = 0 # selects default based on effort #geqo_selection_bias = 2.0 # range 1.5-2.0 #geqo_seed = 0.0 # range 0.0-1.0 # - Other Planner Options - #default_statistics_target = 100 # range 1-10000 #constraint_exclusion = partition # on, off, or partition #cursor_tuple_fraction = 0.1 # range 0.0-1.0 #from_collapse_limit = 8 #join_collapse_limit = 8 # 1 disables collapsing of explicit # JOIN clauses #force_parallel_mode = off #jit = on # allow JIT compilation #plan_cache_mode = auto # auto, force_generic_plan or # force_custom_plan #------------------------------------------------------------------------------ # REPORTING AND LOGGING #------------------------------------------------------------------------------ # - Where to Log - #log_destination = 'stderr' # Valid values are combinations of # stderr, csvlog, syslog, and eventlog, # depending on platform. csvlog # requires logging_collector to be on. # This is used when logging to stderr: #logging_collector = off # Enable capturing of stderr and csvlog # into log files. Required to be on for # csvlogs. # (change requires restart) # These are only used if logging_collector is on: #log_directory = 'log' # directory where log files are written, # can be absolute or relative to PGDATA #log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, # can include strftime() escapes #log_file_mode = 0600 # creation mode for log files, # begin with 0 to use octal notation #log_truncate_on_rotation = off # If on, an existing log file with the # same name as the new log file will be # truncated rather than appended to. # But such truncation only occurs on # time-driven rotation, not on restarts # or size-driven rotation. Default is # off, meaning append to existing files # in all cases. #log_rotation_age = 1d # Automatic rotation of logfiles will # happen after that time. 0 disables. #log_rotation_size = 10MB # Automatic rotation of logfiles will # happen after that much log output. # 0 disables. # These are relevant when logging to syslog: #syslog_facility = 'LOCAL0' #syslog_ident = 'postgres' #syslog_sequence_numbers = on #syslog_split_messages = on # This is only relevant when logging to eventlog (win32): # (change requires restart) #event_source = 'PostgreSQL' # - When to Log - #log_min_messages = warning # values in order of decreasing detail: # debug5 # debug4 # debug3 # debug2 # debug1 # info # notice # warning # error # log # fatal # panic #log_min_error_statement = error # values in order of decreasing detail: # debug5 # debug4 # debug3 # debug2 # debug1 # info # notice # warning # error # log # fatal # panic (effectively off) #log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements # and their durations, > 0 logs only # statements running at least this number # of milliseconds #log_min_duration_sample = -1 # -1 is disabled, 0 logs a sample of statements # and their durations, > 0 logs only a sample of # statements running at least this number # of milliseconds; # sample fraction is determined by log_statement_sample_rate #log_statement_sample_rate = 1.0 # fraction of logged statements exceeding # log_min_duration_sample to be logged; # 1.0 logs all such statements, 0.0 never logs #log_transaction_sample_rate = 0.0 # fraction of transactions whose statements # are logged regardless of their duration; 1.0 logs all # statements from all transactions, 0.0 never logs # - What to Log - #debug_print_parse = off #debug_print_rewritten = off #debug_print_plan = off #debug_pretty_print = on #log_checkpoints = off #log_connections = off #log_disconnections = off #log_duration = off #log_error_verbosity = default # terse, default, or verbose messages #log_hostname = off #log_line_prefix = '%m [%p] ' # special values: # %a = application name # %u = user name # %d = database name # %r = remote host and port # %h = remote host # %b = backend type # %p = process ID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) # %i = command tag # %e = SQL state # %c = session ID # %l = session line number # %s = session start timestamp # %v = virtual transaction ID # %x = transaction ID (0 if none) # %q = stop here in non-session # processes # %% = '%' # e.g. '<%u%%%d> ' #log_lock_waits = off # log lock waits >= deadlock_timeout #log_parameter_max_length = -1 # when logging statements, limit logged # bind-parameter values to N bytes; # -1 means print in full, 0 disables #log_parameter_max_length_on_error = 0 # when logging an error, limit logged # bind-parameter values to N bytes; # -1 means print in full, 0 disables #log_statement = 'none' # none, ddl, mod, all #log_replication_commands = off #log_temp_files = -1 # log temporary files equal or larger # than the specified size in kilobytes; # -1 disables, 0 logs all temp files log_timezone = 'UTC' #------------------------------------------------------------------------------ # PROCESS TITLE #------------------------------------------------------------------------------ #cluster_name = '' # added to process titles if nonempty # (change requires restart) #update_process_title = on #------------------------------------------------------------------------------ # STATISTICS #------------------------------------------------------------------------------ # - Query and Index Statistics Collector - #track_activities = on #track_counts = on #track_io_timing = off #track_functions = none # none, pl, all #track_activity_query_size = 1024 # (change requires restart) #stats_temp_directory = 'pg_stat_tmp' # - Monitoring - #log_parser_stats = off #log_planner_stats = off #log_executor_stats = off #log_statement_stats = off #------------------------------------------------------------------------------ # AUTOVACUUM #------------------------------------------------------------------------------ #autovacuum = on # Enable autovacuum subprocess? 'on' # requires track_counts to also be on. #log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and # their durations, > 0 logs only # actions running at least this number # of milliseconds. #autovacuum_max_workers = 3 # max number of autovacuum subprocesses # (change requires restart) #autovacuum_naptime = 1min # time between autovacuum runs #autovacuum_vacuum_threshold = 50 # min number of row updates before # vacuum #autovacuum_vacuum_insert_threshold = 1000 # min number of row inserts # before vacuum; -1 disables insert # vacuums #autovacuum_analyze_threshold = 50 # min number of row updates before # analyze #autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum #autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table # size before insert vacuum #autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze #autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum # (change requires restart) #autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age # before forced vacuum # (change requires restart) #autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for # autovacuum, in milliseconds; # -1 means use vacuum_cost_delay #autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for # autovacuum, -1 means use # vacuum_cost_limit #------------------------------------------------------------------------------ # CLIENT CONNECTION DEFAULTS #------------------------------------------------------------------------------ # - Statement Behavior - #client_min_messages = notice # values in order of decreasing detail: # debug5 # debug4 # debug3 # debug2 # debug1 # log # notice # warning # error #search_path = '"$user", public' # schema names #row_security = on #default_tablespace = '' # a tablespace name, '' uses the default #temp_tablespaces = '' # a list of tablespace names, '' uses # only default tablespace #default_table_access_method = 'heap' #check_function_bodies = on #default_transaction_isolation = 'read committed' #default_transaction_read_only = off #default_transaction_deferrable = off #session_replication_role = 'origin' #statement_timeout = 0 # in milliseconds, 0 is disabled #lock_timeout = 0 # in milliseconds, 0 is disabled #idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled #vacuum_freeze_min_age = 50000000 #vacuum_freeze_table_age = 150000000 #vacuum_multixact_freeze_min_age = 5000000 #vacuum_multixact_freeze_table_age = 150000000 #vacuum_cleanup_index_scale_factor = 0.1 # fraction of total number of tuples # before index cleanup, 0 always performs # index cleanup #bytea_output = 'hex' # hex, escape #xmlbinary = 'base64' #xmloption = 'content' #gin_fuzzy_search_limit = 0 #gin_pending_list_limit = 4MB # - Locale and Formatting - datestyle = 'iso, mdy' #intervalstyle = 'postgres' timezone = 'UTC' #timezone_abbreviations = 'Default' # Select the set of available time zone # abbreviations. Currently, there are # Default # Australia (historical usage) # India # You can create your own file in # share/timezonesets/. #extra_float_digits = 1 # min -15, max 3; any value >0 actually # selects precise output mode #client_encoding = sql_ascii # actually, defaults to database # encoding # These settings are initialized by initdb, but they can be changed. lc_messages = 'en_US.utf8' # locale for system error message # strings lc_monetary = 'en_US.utf8' # locale for monetary formatting lc_numeric = 'en_US.utf8' # locale for number formatting lc_time = 'en_US.utf8' # locale for time formatting # default configuration for text search default_text_search_config = 'pg_catalog.english' # - Shared Library Preloading - #shared_preload_libraries = '' # (change requires restart) #local_preload_libraries = '' #session_preload_libraries = '' #jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - #dynamic_library_path = '$libdir' #extension_destdir = '' # prepend path when loading extensions # and shared objects (added by Debian) #------------------------------------------------------------------------------ # LOCK MANAGEMENT #------------------------------------------------------------------------------ #deadlock_timeout = 1s #max_locks_per_transaction = 64 # min 10 # (change requires restart) #max_pred_locks_per_transaction = 64 # min 10 # (change requires restart) #max_pred_locks_per_relation = -2 # negative values mean # (max_pred_locks_per_transaction # / -max_pred_locks_per_relation) - 1 #max_pred_locks_per_page = 2 # min 0 #------------------------------------------------------------------------------ # VERSION AND PLATFORM COMPATIBILITY #------------------------------------------------------------------------------ # - Previous PostgreSQL Versions - #array_nulls = on #backslash_quote = safe_encoding # on, off, or safe_encoding #escape_string_warning = on #lo_compat_privileges = off #operator_precedence_warning = off #quote_all_identifiers = off #standard_conforming_strings = on #synchronize_seqscans = on # - Other Platforms and Clients - #transform_null_equals = off #------------------------------------------------------------------------------ # ERROR HANDLING #------------------------------------------------------------------------------ #exit_on_error = off # terminate session on any error? #restart_after_crash = on # reinitialize after backend crash? #data_sync_retry = off # retry or panic on failure to fsync # data? # (change requires restart) #------------------------------------------------------------------------------ # CONFIG FILE INCLUDES #------------------------------------------------------------------------------ # These options allow settings to be loaded from files other than the # default postgresql.conf. Note that these are directives, not variable # assignments, so they can usefully be given more than once. #include_dir = '...' # include files ending in '.conf' from # a directory, e.g., 'conf.d' #include_if_exists = '...' # include file only if it exists #include = '...' # include file #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS #------------------------------------------------------------------------------ # Add settings for extensions here root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# nano postgresql.conf root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# PGDATA="/mnt/Pool2/ix-applications/releases/nextcloud/volumes/ix_volumes/pgData" root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# grep -q "^include_dir = 'conf.d'" "$PGDATA/postgresql.conf" || \ echo "include_dir = 'conf.d'" | sudo tee -a "$PGDATA/postgresql.conf" include_dir = 'conf.d' root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# sudo mkdir -p "$PGDATA/conf.d" root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# sudo tee "$PGDATA/conf.d/20-nextcloud-tuned.conf" >/dev/null <<'EOF' # --- Biohazard VFX tuned defaults for Nextcloud on ZFS --- # Memory shared_buffers = 16GB # restart effective_cache_size = 40GB work_mem = 128MB maintenance_work_mem = 2GB # WAL / checkpoints wal_compression = on max_wal_size = 8GB checkpoint_timeout = 15min checkpoint_completion_target = 0.9 # Storage hints (ZFS/HDD) effective_io_concurrency = 256 random_page_cost = 1.25 # Concurrency / misc max_connections = 200 # restart jit = off huge_pages = off # restart; we disabled THP systemwide EOF root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# cd .. root@truenas[...releases/nextcloud/volumes/ix_volumes]# ls data html pgBackup pgData root@truenas[...releases/nextcloud/volumes/ix_volumes]# cd pgData root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# ls PG_VERSION conf.d pg_commit_ts pg_hba.conf pg_logical pg_notify pg_serial pg_stat pg_subtrans pg_twophase pg_xact postgresql.conf postmaster.pid base global pg_dynshmem pg_ident.conf pg_multixact pg_replslot pg_snapshots pg_stat_tmp pg_tblspc pg_wal postgresql.auto.conf postmaster.opts root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# ls -la total 143 drwx------ 20 netdata docker 27 Oct 1 13:48 . drwxr-xr-x 6 root root 6 Jun 16 2024 .. -rw------- 1 netdata docker 3 Jun 16 2024 PG_VERSION drwx------ 7 netdata docker 7 Aug 30 2024 base drwxr-xr-x 2 root root 3 Oct 1 13:48 conf.d drwx------ 2 netdata docker 60 Oct 1 13:25 global drwx------ 2 netdata docker 2 Jun 16 2024 pg_commit_ts drwx------ 2 netdata docker 2 Jun 16 2024 pg_dynshmem -rw------- 1 netdata docker 4782 Jun 16 2024 pg_hba.conf -rw------- 1 netdata docker 1636 Jun 16 2024 pg_ident.conf drwx------ 4 netdata docker 5 Oct 1 13:45 pg_logical drwx------ 4 netdata docker 4 Jun 16 2024 pg_multixact drwx------ 2 netdata docker 2 Jun 16 2024 pg_notify drwx------ 2 netdata docker 2 Jun 16 2024 pg_replslot drwx------ 2 netdata docker 2 Jun 16 2024 pg_serial drwx------ 2 netdata docker 2 Jun 16 2024 pg_snapshots drwx------ 2 netdata docker 2 Oct 1 13:25 pg_stat drwx------ 2 netdata docker 5 Oct 1 13:49 pg_stat_tmp drwx------ 2 netdata docker 3 Oct 1 03:06 pg_subtrans drwx------ 2 netdata docker 2 Jun 16 2024 pg_tblspc drwx------ 2 netdata docker 2 Jun 16 2024 pg_twophase drwx------ 3 netdata docker 8 Oct 1 13:25 pg_wal drwx------ 2 netdata docker 99 Sep 30 15:41 pg_xact -rw------- 1 netdata docker 88 Jun 16 2024 postgresql.auto.conf -rw------- 1 netdata docker 28058 Oct 1 13:48 postgresql.conf -rw------- 1 netdata docker 92 Oct 1 13:25 postmaster.opts -rw------- 1 netdata docker 94 Oct 1 13:25 postmaster.pid root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# grep -Rni 'redis' /mnt/Pool2/ix-applications/releases/nextcloud/volumes/ix_volumes/html/config/config.php 20: 'memcache.distributed' => '\\OC\\Memcache\\Redis', 21: 'memcache.locking' => '\\OC\\Memcache\\Redis', 22: 'redis' => 24: 'host' => 'nextcloud-redis', root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# ls -la /mnt/Pool2/ix-applications/releases/redis/volumes/ix_volumes 2>/dev/null total 1 drwxr-xr-x 2 root root 2 May 10 02:06 . drwxr-xr-x 3 root root 3 May 10 02:06 .. root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# find /mnt/Pool2 -maxdepth 6 -type f -name 'redis.conf' 2>/dev/null | head -20 ^C root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# k3s kubectl get pods -A | grep redis ix-nextcloud nextcloud-redis-557d99b75-rgj4d 1/1 Running 0 26m root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# k3s kubectl -n ix-redis exec -it nextcloud-redis-557d99b75-rgj4d -- sh Error from server (NotFound): pods "nextcloud-redis-557d99b75-rgj4d" not found root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# k3s kubectl -n ix-redis exec -it ix-nextcloud -- sh Error from server (NotFound): pods "ix-nextcloud" not found root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# k3s kubectl get pods -A | grep redis ix-nextcloud nextcloud-redis-557d99b75-rgj4d 1/1 Running 0 27m root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# POD=nextcloud-redis-557d99b75-rgj4d root@truenas[...s/nextcloud/volumes/ix_volumes/pgData]# k3s kubectl -n ix-nextcloud exec -it $POD -- sh $ $ $ redis-cli INFO | egrep 'redis_version|aof_enabled|maxmemory|maxmemory_policy' $ ^C $ ^C $ ^C $ ^C $ ^[[A^[[A^[[A^[[A^C $ ^C $ ^C $ ^C $ ^C $ ^C $ ^C $ ^C $ ^C $ ^C $ ``` ### Started a new session because the last one crashed ```bash admin@truenas[~]$ sudo su [sudo] password for admin: root@truenas[/home/admin]# POD=nextcloud-redis-557d99b75-rgj4d root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud get pod $POD -o jsonpath='{.spec.containers[0].image}{"\n"}' bitnami/redis:7.0.11 root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it $POD -- sh -lc 'command -v redis-cli || ls -l /opt/bitnami/redis/bin/redis-cli || ls -l /usr/local/bin/redis-cli' -rwxrwxr-x 1 root root 966496 May 20 2023 /opt/bitnami/redis/bin/redis-cli root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud get secrets | grep -i redis nextcloud-redis-creds Opaque 3 156d root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis -o jsonpath='{.data.redis-password}' | base64 -d) Error from server (NotFound): secrets "nextcloud-redis" not found root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ > root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o go-template='{{range $k,$v := .data}}{{printf "%s\n" $k}}{{end}}' ALLOW_EMPTY_PASSWORD REDIS_HOST REDIS_PASSWORD root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.redis-password}' | base64 -d 2>/dev/null || true) [ -z "$REDIS_PASS" ] && REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.password}' | base64 -d 2>/dev/null || true) [ -z "$REDIS_PASS" ] && REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.redis-passwords}' | base64 -d 2>/dev/null || true) root@truenas[/home/admin]# printf 'len=%s\n' "${#REDIS_PASS}" len=0 root@truenas[/home/admin]# POD=nextcloud-redis-557d99b75-rgj4d root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'redis_version|aof_enabled|maxmemory|maxmemory_policy'" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. AUTH failed: WRONGPASS invalid username-password pair or user is disabled. command terminated with exit code 1 root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" CONFIG SET appendonly yes" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. AUTH failed: WRONGPASS invalid username-password pair or user is disabled. (error) NOAUTH Authentication required. ``` `root@truenas[/home/admin]# echo "FUCK"` # FUCK ```bash root@truenas[/home/admin]# ^[[200~REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ cmdsubst> echo "FUCK" root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# printf 'len=%s\n' "${#REDIS_PASS}" len=32 root@truenas[/home/admin]# POD=nextcloud-redis-557d99b75-rgj4d root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc \ "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'redis_version|aof_enabled|maxmemory|maxmemory_policy'" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. redis_version:7.0.11 maxmemory:0 maxmemory_human:0B maxmemory_policy:noeviction aof_enabled:1 root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" CONFIG SET appendonly yes" k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" CONFIG SET appendfsync everysec" k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" CONFIG SET maxmemory 8gb" k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" CONFIG SET maxmemory-policy allkeys-lru" k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" CONFIG REWRITE || echo 'config rewrite not allowed (read-only mount)'" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. OK Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. OK Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. OK Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. OK Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. OK root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it nextcloud-redis-557d99b75-rgj4d -- sh -lc \ "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. maxmemory:8589934592 maxmemory_human:8.00G maxmemory_policy:allkeys-lru aof_enabled:1 ``` `root@truenas[/home/admin]# echo "I just added the REDIS_EXTRA_FLAGS thru the Nextcloud app UI on Truenas"` # I just added the REDIS_EXTRA_FLAGS thru the Nextcloud app UI on Truenas ```bash root@truenas[/home/admin]# POD=$(k3s kubectl -n ix-nextcloud get pods | grep redis | awk '{print $1}') root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. maxmemory:0 maxmemory_human:0B maxmemory_policy:noeviction aof_enabled:1 ``` `root@truenas[/home/admin]# echo "The chart did not accept REDIS_EXTRA_FLAGS"` # The chart did not accept the fucking REDIS_EXTRA_FLAGS ```bash root@truenas[/home/admin]# POD=$(k3s kubectl -n ix-nextcloud get pods | grep redis | awk '{print $1}') root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. maxmemory:0 maxmemory_human:0B maxmemory_policy:noeviction aof_enabled:1 root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud set env deploy/nextcloud-redis \ REDIS_EXTRA_FLAGS="--appendonly yes --appendfsync everysec --maxmemory 8gb --maxmemory-policy allkeys-lru" deployment.apps/nextcloud-redis env updated root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud rollout restart deploy/nextcloud-redis deployment.apps/nextcloud-redis restarted root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud rollout status deploy/nextcloud-redis Waiting for deployment "nextcloud-redis" rollout to finish: 0 of 1 updated replicas are available... deployment "nextcloud-redis" successfully rolled out root@truenas[/home/admin]# POD=$(k3s kubectl -n ix-nextcloud get pods -l app.kubernetes.io/name=redis -o jsonpath='{.items[0].metadata.name}') error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}} root@truenas[/home/admin]# POD=$(k3s kubectl -n ix-nextcloud get pods | grep redis | awk '{print $1}') root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. maxmemory:8589934592 maxmemory_human:8.00G maxmemory_policy:allkeys-lru aof_enabled:1 root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud patch deploy nextcloud-redis --type=json -p='[ {"op":"add","path":"/spec/template/spec/containers/0/args","value": ["--appendonly","yes","--appendfsync","everysec","--maxmemory","8gb","--maxmemory-policy","allkeys-lru"]}]' deployment.apps/nextcloud-redis patched root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud rollout restart deploy/nextcloud-redis deployment.apps/nextcloud-redis restarted root@truenas[/home/admin]# POD=$(k3s kubectl -n ix-nextcloud get pods | grep redis | awk '{print $1}') root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" error: unable to upgrade connection: container not found ("nextcloud") root@truenas[/home/admin]# POD=$(k3s kubectl -n ix-nextcloud get pods | grep redis | awk '{print $1}') root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud get pods | grep redis | awk '{print $1}' nextcloud-redis-698cdbcdb9-bsk77 root@truenas[/home/admin]# POD=$(k3s kubectl -n ix-nextcloud get pods | grep redis | awk '{print $1}') root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds \ -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n ix-nextcloud exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" error: unable to upgrade connection: container not found ("nextcloud") root@truenas[/home/admin]# NS=ix-nextcloud root@truenas[/home/admin]# DEP=nextcloud-redis root@truenas[/home/admin]# k3s kubectl -n $NS set env deploy/$DEP \ REDIS_EXTRA_FLAGS="--appendonly yes --appendfsync everysec --maxmemory 8gb --maxmemory-policy allkeys-lru" root@truenas[/home/admin]# k3s kubectl -n $NS patch deploy $DEP --type=json -p='[ {"op":"add","path":"/spec/template/spec/containers/0/args","value": ["--appendonly","yes","--appendfsync","everysec","--maxmemory","8gb","--maxmemory-policy","allkeys-lru"]}]' || true deployment.apps/nextcloud-redis patched (no change) root@truenas[/home/admin]# NS=ix-nextcloud root@truenas[/home/admin]# POD=$(k3s kubectl -n $NS get pods | awk '/redis/{print $1; exit}') root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" error: unable to upgrade connection: container not found ("nextcloud") root@truenas[/home/admin]# ``` `root@truenas[/home/admin]# echo "I was execing into the wrong container"` # I was execing into the wrong container ```bash root@truenas[/home/admin]# NS=ix-nextcloud root@truenas[/home/admin]# POD=$(k3s kubectl -n $NS get pods | awk '/redis/{print $1; exit}') root@truenas[/home/admin]# k3s kubectl -n $NS get pod "$POD" -o jsonpath='{.spec.containers[*].name}{"\n"}' nextcloud root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -c redis -- sh -lc \ '/opt/bitnami/redis/bin/redis-cli -a "$REDIS_PASS" INFO | egrep "aof_enabled|maxmemory|maxmemory_policy"' Error from server (BadRequest): container redis is not valid for pod nextcloud-redis-698cdbcdb9-bsk77 root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -c redis -- sh -lc 'ps -o args= 1' Error from server (BadRequest): container redis is not valid for pod nextcloud-redis-698cdbcdb9-bsk77 root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -c redis -- sh -lc 'printenv | egrep "^REDIS_|REDIS_EXTRA_FLAGS"' Error from server (BadRequest): container redis is not valid for pod nextcloud-redis-698cdbcdb9-bsk77 root@truenas[/home/admin]# ``` `root@truenas[/home/admin]# echo "TRYING IT AGAIN FUCK MY LIFE"` # TRYING IT AGAIN FUCK MY LIFE ```bash root@truenas[/home/admin]# NS=ix-nextcloud root@truenas[/home/admin]# POD=$(k3s kubectl -n $NS get pods -o name | awk -F/ '/^pod\/nextcloud-redis-/{print $2; exit}') root@truenas[/home/admin]# k3s kubectl -n $NS get pod "$POD" \ -o jsonpath='{.spec.containers[*].name}{"\n"}{.spec.initContainers[*].name}{"\n"}' nextcloud root@truenas[/home/admin]# for C in $(k3s kubectl -n $NS get pod "$POD" -o jsonpath='{.spec.containers[*].name}'); do echo "Trying container: $C" if k3s kubectl -n $NS exec "$POD" -c "$C" -- sh -lc 'test -x /opt/bitnami/redis/bin/redis-cli || test -x /usr/local/bin/redis-cli'; then REDIS_CONTAINER="$C"; break fi done echo "Redis container = ${REDIS_CONTAINER:-}" Trying container: nextcloud error: unable to upgrade connection: container not found ("nextcloud") Redis container = root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -c "$REDIS_CONTAINER" -- sh -lc \ '/opt/bitnami/redis/bin/redis-cli -a "$REDIS_PASS" INFO | egrep "aof_enabled|maxmemory|maxmemory_policy"' error: unable to upgrade connection: container not found ("nextcloud") root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -c "$REDIS_CONTAINER" -- sh -lc 'ps -o args= 1' error: unable to upgrade connection: container not found ("nextcloud") root@truenas[/home/admin]# ``` # okay the issue actually is that redis isn't starting: ``` 2025-10-01 14:28:27.510316-07:00redis 21:28:27.51  2025-10-01 14:28:27.511366-07:00redis 21:28:27.51 Welcome to the Bitnami redis container 2025-10-01 14:28:27.512411-07:00redis 21:28:27.51 Subscribe to project updates by watching https://github.com/bitnami/containers 2025-10-01 14:28:27.513445-07:00redis 21:28:27.51 Submit issues and feature requests at https://github.com/bitnami/containers/issues 2025-10-01 14:28:27.514453-07:00redis 21:28:27.51  2025-10-01 14:28:27.514530-07:002025-10-01T14:28:27.514530815-07:00 2025-10-01 14:28:27.514550-07:00/opt/bitnami/scripts/redis/entrypoint.sh: line 28: exec: --: invalid option 2025-10-01 14:28:27.514567-07:00exec: usage: exec [-cl] [-a name] [command [argument ...]] [redirection ...] ``` `root@truenas[/home/admin]# echo "REMOVING THE BAD ARGUMENTS THAT STOPPED REDIS FROM STARTING"` # REMOVING THE BAD ARGUMENTS THAT STOPPED REDIS FROM STARTING ```bash root@truenas[/home/admin]# NS=ix-nextcloud root@truenas[/home/admin]# DEP=nextcloud-redis root@truenas[/home/admin]# k3s kubectl -n $NS patch deploy $DEP --type=json -p='[ {"op":"remove","path":"/spec/template/spec/containers/0/args"} ]' || true deployment.apps/nextcloud-redis patched root@truenas[/home/admin]# k3s kubectl -n $NS set env deploy/$DEP \ REDIS_APPENDONLY=yes \ REDIS_APPENDFSYNC=everysec \ REDIS_MAXMEMORY=8gb \ REDIS_MAXMEMORY_POLICY=allkeys-lru deployment.apps/nextcloud-redis env updated root@truenas[/home/admin]# k3s kubectl -n $NS rollout restart deploy/$DEP k3s kubectl -n $NS rollout status deploy/$DEP deployment.apps/nextcloud-redis restarted Waiting for deployment "nextcloud-redis" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-redis" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-redis" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-redis" rollout to finish: 0 of 1 updated replicas are available... deployment "nextcloud-redis" successfully rolled out root@truenas[/home/admin]# POD=$(k3s kubectl -n $NS get pods | awk '/nextcloud-redis/{print $1; exit}') root@truenas[/home/admin]# REDIS_PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -- sh -lc '/opt/bitnami/redis/bin/redis-cli -a "$REDIS_PASS" INFO | egrep "aof_enabled|maxmemory|maxmemory_policy" || true' Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. AUTH failed: WRONGPASS invalid username-password pair or user is disabled. root@truenas[/home/admin]# root@truenas[/home/admin]# echo "just double quotes lol" just double quotes lol root@truenas[/home/admin]# NS=ix-nextcloud POD=$(k3s kubectl -n $NS get pods | awk '/nextcloud-redis/{print $1; exit}') REDIS_PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -- sh -lc "/opt/bitnami/redis/bin/redis-cli -a \"$REDIS_PASS\" INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. maxmemory:8589934592 maxmemory_human:8.00G maxmemory_policy:allkeys-lru aof_enabled:1 root@truenas[/home/admin]# root@truenas[/home/admin]# cat >/root/reapply-redis-tuning.sh <<'EOF' NS=ix-nextcloud DEP=nextcloud-redis k3s kubectl -n $NS set env deploy/$DEP \ REDIS_APPENDONLY=yes \ REDIS_APPENDFSYNC=everysec \ REDIS_MAXMEMORY=8gb \ REDIS_MAXMEMORY_POLICY=allkeys-lru k3s kubectl -n $NS rollout restart deploy/$DEP EOF root@truenas[/home/admin]# chmod +x /root/reapply-redis-tuning.sh root@truenas[/home/admin]# NS=ix-nextcloud NC=$(k3s kubectl -n $NS get pods | awk '/nextcloud-[^r].*Running/{print $1; exit}') k3s kubectl -n $NS exec -it "$NC" -- php -f /var/www/html/occ config:system:get memcache.locking k3s kubectl -n $NS exec -it "$NC" -- php -f /var/www/html/occ background:status error: Internal error occurred: error executing command in container: failed to exec in container: failed to start exec "30f0b951f4cc2eec0227473409c46658a431fb3eff1ee69133ec0511f0016722": OCI runtime exec failed: exec failed: unable to start container process: exec: "php": executable file not found in $PATH: unknown error: Internal error occurred: error executing command in container: failed to exec in container: failed to start exec "a75d369df03c112c2c128e24c09d044c273ffc74be9c2109c1286f997329758c": OCI runtime exec failed: exec failed: unable to start container process: exec: "php": executable file not found in $PATH: unknown root@truenas[/home/admin]# NS=ix-nextcloud POD=$(k3s kubectl -n $NS get pods | awk '/nextcloud-redis/{print $1; exit}') REDIS_PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) k3s kubectl -n $NS exec -it "$POD" -- sh -lc "REDISCLI_AUTH='$REDIS_PASS' /opt/bitnami/redis/bin/redis-cli INFO | egrep 'aof_enabled|maxmemory|maxmemory_policy'" maxmemory:8589934592 maxmemory_human:8.00G maxmemory_policy:allkeys-lru aof_enabled:1 root@truenas[/home/admin]# PG=$(k3s kubectl -n $NS get pods | awk '/postgres/{print $1; exit}') k3s kubectl -n $NS exec -it "$PG" -- psql -U postgres -Atc "show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;" psql: error: FATAL: role "postgres" does not exist command terminated with exit code 2 root@truenas[/home/admin]# root@truenas[/home/admin]# echo "once again i have exec'd into the wrong container" once again i have exec'd into the wrong container root@truenas[/home/admin]# NS=ix-nextcloud root@truenas[/home/admin]# NCPOD=$(k3s kubectl -n $NS get pods -o name | sed -n 's|pod/||p' | grep -E '^nextcloud-[0-9a-z]+' | head -1) root@truenas[/home/admin]# k3s kubectl -n $NS get pod "$NCPOD" -o jsonpath='{.spec.containers[*].name}{"\n"}' nextcloud root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$NCPOD" -- sh -lc 'command -v php || command -v /opt/bitnami/php/bin/php || true; ls -l /var/www/html/occ || ls -l /bitnami/nextcloud/occ || true' ls: cannot access '/var/www/html/occ': No such file or directory ls: cannot access '/bitnami/nextcloud/occ': No such file or directory root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$NCPOD" -- sh -lc '/opt/bitnami/php/bin/php -v' k3s kubectl -n $NS exec -it "$NCPOD" -- sh -lc '/opt/bitnami/php/bin/php -f /var/www/html/occ config:system:get memcache.locking' k3s kubectl -n $NS exec -it "$NCPOD" -- sh -lc '/opt/bitnami/php/bin/php -f /var/www/html/occ background:status' sh: 1: /opt/bitnami/php/bin/php: not found command terminated with exit code 127 sh: 1: /opt/bitnami/php/bin/php: not found command terminated with exit code 127 sh: 1: /opt/bitnami/php/bin/php: not found command terminated with exit code 127 root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$NCPOD" -- sh -lc "grep -E \"'db(host|name|user|password)'\" -n /var/www/html/config/config.php" grep: /var/www/html/config/config.php: No such file or directory command terminated with exit code 2 root@truenas[/home/admin]# k3s kubectl -n $NS get secrets | grep -i postgre nextcloud-postgres-creds Opaque 5 156d root@truenas[/home/admin]# SEC=Opaque root@truenas[/home/admin]# k3s kubectl -n $NS get secret "$SEC" -o go-template='{{range $k,$v := .data}}{{printf "%s\n" $k}}{{end}}' Error from server (NotFound): secrets "Opaque" not found root@truenas[/home/admin]# SEC=nextcloud-postgres-creds root@truenas[/home/admin]# k3s kubectl -n $NS get secret "$SEC" -o go-template='{{range $k,$v := .data}}{{printf "%s\n" $k}}{{end}}' POSTGRES_DB POSTGRES_HOST POSTGRES_PASSWORD POSTGRES_URL POSTGRES_USER root@truenas[/home/admin]# DBUSER=$(k3s kubectl -n $NS get secret "$SEC" -o jsonpath='{.data.POSTGRESQL_USERNAME}' | base64 -d 2>/dev/null || true) DBPASS=$(k3s kubectl -n $NS get secret "$SEC" -o jsonpath='{.data.POSTGRESQL_PASSWORD}' | base64 -d 2>/dev/null || true) DBNAME=$(k3s kubectl -n $NS get secret "$SEC" -o jsonpath='{.data.database}' | base64 -d 2>/dev/null || true) root@truenas[/home/admin]# HOST=$(k3s kubectl -n $NS get svc -o name | sed -n 's|service/||p' | grep -E 'postgres|pgsql' | head -1) root@nextcloud-postgres[/home/admin]# k3s kubectl -n $NS run pg-client --rm -it --image=bitnami/postgresql:16 --restart=Never -- \ bash -lc "PGPASSWORD='$DBPASS' psql -h $HOST -U '$DBUSER' -d '$DBNAME' -Atc \ \"show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;\"" ``` `root@nextcloud-postgres[/home/admin]# echo "that took too long and i had to quit"` ### that took too long and i had to quit `root@nextcloud-postgres[/home/admin]# echo "trying option A"` # trying option A ``` root@nextcloud-postgres[/home/admin]# NS=ix-nextcloud SEC=nextcloud-postgres-creds DBUSER=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_USER}' | base64 -d) DBPASS=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d) DBNAME=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_DB}' | base64 -d) HOST=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_HOST}' | base64 -d) k3s kubectl -n $NS delete pod pg-client --force --grace-period=0 2>/dev/null || true NAME=pgc-$(date +%s) k3s kubectl -n $NS run $NAME --rm -it --image=bitnami/postgresql:16 --restart=Never -- \ bash -lc "export PGCONNECT_TIMEOUT=5; PGPASSWORD='$DBPASS' psql -h '$HOST' -U '$DBUSER' -d '$DBNAME' -Atc \ \"show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;\"" pod "pg-client" force deleted pod "pgc-1759355097" deleted error: timed out waiting for the condition root@nextcloud-postgres[/home/admin]# ### execing directly into the postgres pod to run commands instead ```bash $ command -v psql || command -v /opt/bitnami/postgresql/bin/psql /usr/bin/psql $ printenv | egrep 'POSTGRES_(USER|PASSWORD|DB)' POSTGRES_PASSWORD=wWVRyeHc4FMi20aiROULKn60GtRoDWZc POSTGRES_USER=nextcloud POSTGRES_DB=nextcloud $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc \ "show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;"> 1.25 $ ``` That was a line continuation error: ```bash $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc \ "show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;"> 1.25 $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc \ "show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;" > 1.25 $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc " select 'shared_buffers='||setting from pg_settings where name='shared_buffers' union all select 'effective_cache_size='||setting from pg_settings where name> ='effective_cache_size' union all select 'wal_compression='||setting from pg_settings where name='wal_compression' union all select 'ma> x_wal_size='||setting from pg_settings where name='max_wal_size' union all select 'random_page_cost='||setting from pg_settings > where name='random_page_cost';> > > > > > > ^C $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc " select 'shared_buffers='||setting from pg_settings where name='shared_buffers' union all select 'effective_cache_size='||setting from pg_settings where nam> e='effective_cache_size' union all select 'wal_compression='||setting from pg_set> tings where name='wal_compression' union all select '> max_wal_size='||setting from pg_settings wher> e name='max_wal_size' union all select 'random_page_cost='||setting from pg_settings where name='random_page_cost';> " shared_buffers=131072 effective_cache_size=5242880 wal_compression=on max_wal_size=8192 random_page_cost=1.25 $ psql -Atc "select pg_reload_conf();" psql: error: FATAL: role "postgres" does not exist $ ``` ### Going back to the host 4/5 are already correct. Only shared_buffers is wrong. shared_buffers=131072 → that’s pages, 131072×8KB = 1GB. effective_cache_size=5242880 → 40GB ✅ wal_compression=on ✅ max_wal_size=8192 → 8GB ✅ random_page_cost=1.25 ✅ shared_buffers needs a restart to take effect, and the postgresql.conf still has the default. ```bash admin@truenas[~]$ CONF="/mnt/Pool2/ix-applications/releases/nextcloud/volumes/ix_volumes/pgData/postgresql.conf" sudo sed -i 's/^shared_buffers.*/shared_buffers = 16GB/' "$CONF" [sudo] password for admin: admin@truenas[~]$ NS=ix-nextcloud PGDEP=$(k3s kubectl -n $NS get deploy -o name | grep -Ei 'postgre|pgsql' | head -1) k3s kubectl -n $NS rollout restart "$PGDEP" k3s kubectl -n $NS rollout status "$PGDEP" WARN[0000] Unable to read /etc/rancher/k3s/k3s.yaml, please start server with --write-kubeconfig-mode to modify kube config permissions error: error loading config file "/etc/rancher/k3s/k3s.yaml": open /etc/rancher/k3s/k3s.yaml: permission denied WARN[0000] Unable to read /etc/rancher/k3s/k3s.yaml, please start server with --write-kubeconfig-mode to modify kube config permissions error: error loading config file "/etc/rancher/k3s/k3s.yaml": open /etc/rancher/k3s/k3s.yaml: permission denied WARN[0000] Unable to read /etc/rancher/k3s/k3s.yaml, please start server with --write-kubeconfig-mode to modify kube config permissions error: error loading config file "/etc/rancher/k3s/k3s.yaml": open /etc/rancher/k3s/k3s.yaml: permission denied admin@truenas[~]$ sudo su root@truenas[/home/admin]# NS=ix-nextcloud PGDEP=$(k3s kubectl -n $NS get deploy -o name | grep -Ei 'postgre|pgsql' | head -1) k3s kubectl -n $NS rollout restart "$PGDEP" k3s kubectl -n $NS rollout status "$PGDEP" deployment.apps/nextcloud-postgres restarted Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 of 1 updated replicas are available... deployment "nextcloud-postgres" successfully rolled out ``` **Going back to the postgres pod to do a healthcheck** ```bash $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;" 1.25 $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc /usr/lib/postgresql/13/bin/psql: option requires an argument -- 'c' Try "psql --help" for more information. $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;" 1.25 $ psql -Atc "show config_file; show data_directory;" grep -n '^shared_buffers' /bitnami/postgresql/data/postgresql.conf 2>/dev/null || truepsql: error: FATAL: role "postgres" does not exist $ $ psql -Atc "show config_file; show data_directory;" psql: error: FATAL: role "postgres" does not exist $ ``` (didnt go well) ```bash PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \ "alter system set shared_buffers = '16GB';"> ALTER SYSTEM $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "alter system set shared_buffers = '16GB';" ALTER SYSTEM ``` **now going back to the host to restart postgres** ```bash root@truenas[/home/admin]# NS=ix-nextcloud k3s kubectl -n $NS rollout restart deploy/nextcloud-postgres k3s kubectl -n $NS rollout status deploy/nextcloud-postgres deployment.apps/nextcloud-postgres restarted Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 of 1 updated replicas are available... deployment "nextcloud-postgres" successfully rolled out root@truenas[/home/admin]# ``` **now going back to postgres pod** ```bash $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;" 1.25 $ ``` (fuck) something is still forcing shared_buffers back to 1GB at startup ```bash $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc \ "select name,setting,unit,source,sourcefile,pending_restart from pg_settings where name in ('shared_buffers','effective_cache_size','wal_compression','max_wa> l_size','random_page_cost');" effective_cache_size|5242880|8kB|configuration file|/var/lib/postgresql/data/conf.d/20-nextcloud-tuned.conf|f max_wal_size|8192|MB|configuration file|/var/lib/postgresql/data/conf.d/20-nextcloud-tuned.conf|f random_page_cost|1.25||configuration file|/var/lib/postgresql/data/conf.d/20-nextcloud-tuned.conf|f shared_buffers|131072|8kB|command line||f wal_compression|on||configuration file|/var/lib/postgresql/data/conf.d/20-nextcloud-tuned.conf|f ``` ```bash $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "show config_file; show data_directory;" ps -o args= 1 | sed 's/ -/ \n-/g' | sed -n '1,120p'/var/lib/postgresql/data $ /bin/sh: 5: ps: not found $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "show config_file; show data_directory;" ps -o args= 1 | sed 's/ -/ \n-/g' | sed -n '1,120p'/var/lib/postgresql/data $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "show config_file; show data_directory;" /bin/sh: 7: ps: not found sed: invalid option -- 'h' Usage: sed [OPTION]... {script-only-if-no-other-script} [input-file]... -n, --quiet, --silent suppress automatic printing of pattern space --debug annotate program execution -e script, --expression=script add the script to the commands to be executed -f script-file, --file=script-file add the contents of script-file to the commands to be executed --follow-symlinks follow symlinks when processing in place -i[SUFFIX], --in-place[=SUFFIX] edit files in place (makes backup if SUFFIX supplied) -l N, --line-length=N specify the desired line-wrap length for the `l' command --posix disable all GNU extensions. -E, -r, --regexp-extended use extended regular expressions in the script (for portability use POSIX -E). -s, --separate consider files as separate rather than as a single, continuous long stream. --sandbox operate in sandbox mode (disable e/r/w commands). -u, --unbuffered load minimal amounts of data from the input files and flush the output buffers more often -z, --null-data separate lines by NUL characters --help display this help and exit --version output version information and exit If no -e, --expression, -f, or --file option is given, then the first non-option argument is taken as the sed script to interpret. All remaining arguments are names of input files; if no input files are specified, then the standard input is read. GNU sed home page: . General help using GNU software: . $ ps -o args= 1 | sed 's/ -/ \n-/g' | sed -n '1,120p' /bin/sh: 8: ps: not found $ ``` **Going back to the host to remove the command-line override in the deployment.** ```bash root@truenas[/home/admin]# NS=ix-nextcloud DEP=nextcloud-postgres root@truenas[/home/admin]# k3s kubectl -n $NS get deploy $DEP -o jsonpath='{.spec.template.spec.containers[0].command}{"\n"}{.spec.template.spec.containers[0].args}{"\n"}' ["-c","max_connections=500","-c","shared_buffers=1024MB"] root@truenas[/home/admin]# k3s kubectl -n $NS patch deploy $DEP --type=json -p='[ {"op":"remove","path":"/spec/template/spec/containers/0/args"} ]' || true deployment.apps/nextcloud-postgres patched root@truenas[/home/admin]# k3s kubectl -n $NS patch deploy $DEP --type=json -p='[ {"op":"remove","path":"/spec/template/spec/containers/0/command"} ]' || true The request is invalid: the server rejected our request due to an error in our request root@truenas[/home/admin]# k3s kubectl -n $NS rollout restart deploy/$DEP k3s kubectl -n $NS rollout status deploy/$DEP deployment.apps/nextcloud-postgres restarted Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 of 1 updated replicas are available... deployment "nextcloud-postgres" successfully rolled out root@truenas[/home/admin]# ``` **going back inside the postgres pod:** ```bash $ PGPASSWORD="$POSTGRES_PASSWORD" psql -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc \ "show shared_buffers; show effective_cache_size; show wal_compression; show max_wal_size; show random_page_cost;"> 1.25 ``` this is starting to get very repetitive The Postgres issue is simpler: the pod is still starting postgres with -c shared_buffers=1024MB. That always wins over postgresql.conf, conf.d, and ALTER SYSTEM, which is why pg_settings.source = 'command line'. Fix is to either remove the CLI flags for good or override with values i want. ```bash root@truenas[/home/admin]# NS=ix-nextcloud root@truenas[/home/admin]# DEP=nextcloud-postgres root@truenas[/home/admin]# k3s kubectl -n $NS get deploy $DEP -o jsonpath='{.spec.template.spec.containers[0].command}{"\n"}{.spec.template.spec.containers[0].args}{"\n"}' root@truenas[/home/admin]# k3s kubectl -n $NS get deploy $DEP -o yaml | grep -nE 'initContainers|containers:|command:|args:|-c|shared_buffers|max_connections' 52: containers: 70: name: nextcloud-postgres-creds 75: command: 77: - -c 91: command: 93: - -c 121: command: 123: - -c root@truenas[/home/admin]# k3s kubectl -n $NS patch deploy $DEP --type=json -p='[ {"op":"add","path":"/spec/template/spec/containers/0/args","value": ["-c","shared_buffers=16GB", "-c","max_connections=200", "-c","wal_compression=on", "-c","max_wal_size=8GB", "-c","random_page_cost=1.25"]}]' deployment.apps/nextcloud-postgres patched root@truenas[/home/admin]# k3s kubectl -n $NS rollout restart deploy/$DEP k3s kubectl -n $NS rollout status deploy/$DEP deployment.apps/nextcloud-postgres restarted Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 of 1 updated replicas are available... deployment "nextcloud-postgres" successfully rolled out root@truenas[/home/admin]# POD=$(k3s kubectl -n $NS get pods -o name | sed -n 's|pod/||p' | grep -E '^nextcloud-postgres' | head -1) root@truenas[/home/admin]# SEC=nextcloud-postgres-creds root@truenas[/home/admin]# DBUSER=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_USER}' | base64 -d) root@truenas[/home/admin]# DBPASS=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d) root@truenas[/home/admin]# DBNAME=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_DB}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -- bash -lc \ "PGPASSWORD='$DBPASS' psql -h 127.0.0.1 -U '$DBUSER' -d '$DBNAME' -Atc \ \"select name,setting,unit,source,coalesce(sourcefile,'') from pg_settings where name in ('shared_buffers','effective_cache_size','wal_compression','max_wal_size','random_page_cost') order by name;\"" effective_cache_size|5242880|8kB|configuration file|/var/lib/postgresql/data/conf.d/20-nextcloud-tuned.conf max_wal_size|8192|MB|command line| random_page_cost|1.25||command line| shared_buffers|2097152|8kB|command line| wal_compression|on||command line| root@truenas[/home/admin]# k3s kubectl -n $NS get deploy $DEP -o yaml | grep -n 'shared_buffers' 55: - shared_buffers=16GB root@truenas[/home/admin]# k3s kubectl -n $NS rollout restart deploy/$DEP k3s kubectl -n $NS rollout status deploy/$DEP k3s kubectl -n $NS exec -it "$POD" -- bash -lc \ "PGPASSWORD='$DBPASS' psql -h 127.0.0.1 -U '$DBUSER' -d '$DBNAME' -Atc \ \"select name,setting,unit,source from pg_settings where name='shared_buffers';\"" deployment.apps/nextcloud-postgres restarted Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 out of 1 new replicas have been updated... Waiting for deployment "nextcloud-postgres" rollout to finish: 0 of 1 updated replicas are available... deployment "nextcloud-postgres" successfully rolled out Error from server (NotFound): pods "nextcloud-postgres-bfd949b4d-4h468" not found root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -- bash -lc \ "PGPASSWORD='$DBPASS' psql -h 127.0.0.1 -U '$DBUSER' -d '$DBNAME' -Atc \ \"select name,setting,unit,source from pg_settings where name='shared_buffers';\"" Error from server (NotFound): pods "nextcloud-postgres-bfd949b4d-4h468" not found root@truenas[/home/admin]# POD=$(k3s kubectl -n $NS get pods -o name | sed -n 's|pod/||p' | grep -E '^nextcloud-postgres' | head -1) root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -- bash -lc \ "PGPASSWORD='$DBPASS' psql -h 127.0.0.1 -U '$DBUSER' -d '$DBNAME' -Atc \ \"select name,setting,unit,source from pg_settings where name='shared_buffers';\"" shared_buffers|2097152|8kB|command line root@truenas[/home/admin]# k3s kubectl -n $NS get deploy $DEP -o jsonpath='{.spec.template.spec.containers[0].resources}{"\n"}' {"limits":{"cpu":"4","memory":"12Gi"},"requests":{"cpu":"10m","memory":"50Mi"}} root@truenas[/home/admin]# ``` I raised the limit in the apps ui interface to 24GiB so there should be no issues now. Going to verify: ```bash root@truenas[/home/admin]# POD=$(k3s kubectl -n $NS get pods -o name | sed -n 's|pod/||p' | grep -E '^nextcloud-postgres' | head -1) SEC=nextcloud-postgres-creds DBUSER=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_USER}' | base64 -d) DBPASS=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_PASSWORD}' | base64 -d) DBNAME=$(k3s kubectl -n $NS get secret $SEC -o jsonpath='{.data.POSTGRES_DB}' | base64 -d) k3s kubectl -n $NS exec -it "$POD" -- bash -lc \ "PGPASSWORD='$DBPASS' psql -h 127.0.0.1 -U '$DBUSER' -d '$DBNAME' -Atc \ \"select name,setting,unit,source from pg_settings where name in ('shared_buffers','effective_cache_size','wal_compression','max_wal_size','random_page_cost') order by name;\"" effective_cache_size|5242880|8kB|configuration file max_wal_size|8192|MB|command line random_page_cost|1.25||command line shared_buffers|2097152|8kB|command line wal_compression|on||command line root@truenas[/home/admin]# ``` Fuck yeah, confirming it actually is applied to the running pod: ```bash root@truenas[/home/admin]# NS=ix-nextcloud POD=$(k3s kubectl -n $NS get pods -o name | sed -n 's|pod/||p' | grep -E '^nextcloud-postgres' | head -1) k3s kubectl -n $NS exec "$POD" -- sh -lc 'test -f /sys/fs/cgroup/memory.max && cat /sys/fs/cgroup/memory.max || cat /sys/fs/cgroup/memory/memory.limit_in_bytes' 25769803776 root@truenas[/home/admin]# ``` Making sure huge pages is off: ```bash root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -- bash -lc "psql -Atc \"show huge_pages;\" -U '$DBUSER' -h 127.0.0.1 -d '$DBNAME'" # should be off (as you set) off zsh: number expected ``` another health check ```bash k3s kubectl -n $NS exec -it "$POD" -- bash -lc \ "psql -Atc \"select now(), pg_is_in_recovery(), current_setting('shared_buffers'), current_setting('max_wal_size');\" -U '$DBUSER' -h 127.0.0.1 -d '$DBNAME'" ``` Now I'm going to do a check on Redis to make sure thats good. ```bash root@truenas[/home/admin]# NS=ix-nextcloud POD=$(k3s kubectl -n $NS get pods -o name | sed -n 's|pod/||p' | grep -E '^nextcloud-redis' | head -1) REDIS_PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) root@truenas[/home/admin]# k3s kubectl -n $NS exec -it "$POD" -- sh -lc " export REDISCLI_AUTH='$REDIS_PASS'; dquote> # 1) Liveness /opt/bitnami/redis/bin/redis-cli PING; # 2) Config + key metrics /opt/bitnami/redis/bin/redis-cli --no-auth-warning \ CONFIG GET appendonly \ CONFIG GET appendfsync \ CONFIG GET maxmemory \ CONFIG GET maxmemory-policy \ INFO server \ INFO memory \ INFO clients \ INFO persistence \ INFO stats \ INFO replication \ | egrep -i 'redis_version|maxmemory:|maxmemory_policy|aof_enabled|appendonly|appendfsync|used_memory_human|role:|connected_clients|evicted_keys|keyspace_hits|keyspace_misses|instantaneous_ops_per_sec|rdb_bgsave_in_progress|aof_rewrite_in_progress'; # 3) Latency + slowlog /opt/bitnami/redis/bin/redis-cli LATENCY DOCTOR; /opt/bitnami/redis/bin/redis-cli SLOWLOG LEN; # 4) Write test /opt/bitnami/redis/bin/redis-cli SET __hc:ts \$(date +%s) EX 60; /opt/bitnami/redis/bin/redis-cli GET __hc:ts; # 5) Who am I (ACL sanity) /opt/bitnami/redis/bin/redis-cli ACL WHOAMI; " PONG appendfsync appendonly I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this Redis instance. You may use "CONFIG SET latency-monitor-threshold ." in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at https://redis.io/topics/latency-monitor. (integer) 0 OK "1759357659" "default" root@truenas[/home/admin]# ``` FUCKING HAL????? few more things: ```bash root@truenas[/home/admin]# NS=ix-nextcloud; POD=$(k3s kubectl -n $NS get pods -o name | sed -n 's|pod/||p' | grep -E '^nextcloud-redis' | head -1); \ PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d); \ k3s kubectl -n $NS exec -it "$POD" -- sh -lc "REDISCLI_AUTH='$PASS' /opt/bitnami/redis/bin/redis-cli INFO | egrep -i 'role:|connected_clients|used_memory_human|maxmemory_human|maxmemory_policy|aof_enabled|aof_last_write_status|evicted_keys|instantaneous_ops_per_sec'" connected_clients:11 used_memory_human:1.46M maxmemory_human:8.00G maxmemory_policy:allkeys-lru aof_enabled:1 aof_last_write_status:ok instantaneous_ops_per_sec:95 evicted_keys:0 role:master root@truenas[/home/admin]# watch -n5 "k3s kubectl -n ix-nextcloud exec -it $(k3s kubectl -n ix-nextcloud get pods -o name | sed -n 's|pod/||p' | grep ^nextcloud-redis | head -1) -- sh -lc \ \"REDISCLI_AUTH=\$(k3s kubectl -n ix-nextcloud get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}'|base64 -d) \ /opt/bitnami/redis/bin/redis-cli INFO | egrep -i 'used_memory_human|maxmemory_human|evicted_keys|instantaneous_ops_per_sec|aof_last_write_status'\"" root@truenas[/home/admin]# NS=ix-nextcloud POD=$(k3s kubectl -n $NS get pods -o name | sed -n 's|pod/||p' | grep ^nextcloud-redis | head -1) PASS=$(k3s kubectl -n $NS get secret nextcloud-redis-creds -o jsonpath='{.data.REDIS_PASSWORD}' | base64 -d) k3s kubectl -n $NS exec "$POD" -- sh -lc " REDISCLI_AUTH='$PASS' /opt/bitnami/redis/bin/redis-cli INFO | awk -F: ' /^used_memory_human/ {mem=\$2} /^maxmemory_human/ {max=\$2} /^aof_last_write_status/ {aof=\$2} /^evicted_keys/ {ev=\$2} /^instantaneous_ops_per_sec/ {ops=\$2} END {gsub(/^[ \t]+|[ \t]+$/, \"\", mem); gsub(/^[ \t]+|[ \t]+$/, \"\", max); gsub(/^[ \t]+|[ \t]+$/, \"\", aof); print \"mem=\" mem, \"max=\" max, \"aof=\" aof, \"ops=\" ops, \"evicted=\" ev }' " evicted=0 root@truenas[/home/admin]# ```