TOMODACHI SHELL

Free Palestine !! - Free Gaza !!

readonly ROOT_CHECK_DATA="${BACKGROUND_DATA}/root_disk_check.data";
readonly BACKUP_CHECK_DATA="${BACKGROUND_DATA}/backup_disk_check.data";
readonly NO_BACKUP_MOUNT="/backup not mounted";
readonly ROOT_CHECK_ERRORS="${BACKGROUND_DATA}/root_disk_check.errors";
readonly BACKUP_CHECK_ERRORS="${BACKGROUND_DATA}/backup_disk_check.errors";

file_type_check(){
        local files=$(cat);
        for f in $files; do
                file "$f";
        done;
}

file_du(){
        local files=$(cat);
        for f in $files; do
                du -b "$f";
        done;
}

nginx_cache_check(){
        local nginx_cache="/var/cache/nginx";
        local proxy_tmp="proxy_temp";
        local nginx_proxy_tmp="/var/cache/nginx/$proxy_tmp";
        local disk_size_in_k=$(df -BK / | tail -1 | awk '{print $2}' | grep -oP "[0-9]+");
        local nginx_cache_size_in_k=$(du -sBK $nginx_proxy_tmp | grep -oP [0-9]+);
        local nginx_cache_max_size_threshold=$(awk '{printf "%d", $1 / 10}' <<< $disk_size_in_k);

        if [[ $nginx_cache_size_in_k -gt $nginx_cache_max_size_threshold ]]; then
                find $nginx_proxy_tmp -maxdepth 1 -not -name "$proxy_tmp" -exec rm -rf {} \;;
                # Check if auto fix was successful
                disk_usage_percentage=$(df $part | tail -n +2 | awk '{print $5}' | grep -oE ^[0-9]+);
                if [[ $disk_usage_percentage -lt $threshold_warning ]]; then
                        part_du_data[auto_fix_status]=$AUTO_FIX_SUCCESS;
                        part_du_data[service_status]=$SERVICE_ACTIVE;
                else
                        part_du_data[auto_fix_status]=$AUTO_FIX_FAIL;
                fi;
        fi;
}

extra_root_disk_checks(){
        nginx_cache_check;
}

fully_managed_root_tmp_manual_review(){
        # When the ${part_du_data[auto_fix_status]} variable is set to 'no_auto_fix', the client will be
        # automatically notified, however, cases where the /tmp partition is in the warning/critical state, and there
        # is no auto-fix, manual review is required, the same applies to cases when both / and /tmp partitions in the
        # warning/critical state. The absense of the variable equates to 'N/A', which means manual review.

        if [[ $(management_type_check) == $COMPLETE ]]; then
                if [[ ${part_critical_or_warning[/]} && ${part_critical_or_warning[/tmp]} && ${part_du_data[auto_fix_status]} == $AUTO_FIX_NOPE ]]; then
                        unset part_du_data[auto_fix_status];
                elif [[ ${part_critical_or_warning[/tmp]} && ${part_du_data[auto_fix_status]} == $AUTO_FIX_NOPE ]]; then
                        unset part_du_data[auto_fix_status];
                fi;
        fi;
}

get_total_sql_f_size_m_blocks(){
        for sql_f_size in $(find /tmp -type f -name "#sql-temptable-*.MAD" | file_du |
                grep -oP "^[0-9]+"); do
                total_sql_f_size=$(($total_sql_f_size + $sql_f_size));
        done;

        if [[ -n $total_sql_f_size ]]; then
                total_sql_f_size_m_blocks=$(($total_sql_f_size / 1024 / 1024));
        fi;
}

mysql_tmp_files(){
        local total_sql_f_size;
        local total_sql_f_size_m_blocks=0;
        local tmp_m_block_capacity=$(df -B M /tmp | tail -n +2 | awk '{print $2}' | grep -oP '^[0-9]+');

        get_total_sql_f_size_m_blocks;

        if [[ $total_sql_f_size_m_blocks -ge $(( $tmp_m_block_capacity / 2 )) ]]; then
                # stop the mysql service
                /scripts/restartsrv_mysql --stop 1>/dev/null 2>&1;
                # Delete the leftover mysql tmp files, in case the mysql service crashed and failed to remove them
                find /tmp -type f -name "#sql-temptable-*.MAD" -delete -or -name "#sql-temptable-*.MAI" -delete 2>/dev/null;
                # create a new tmp directory for the mysql service
                local mysqltmp="/var/lib/mysql/tmp";
                local mysql_conf="/etc/my.cnf";
                mkdir $mysqltmp;
                chown mysql: $mysqltmp;
                sed -i "s:\[mysqld\]:&\ntmpdir=$mysqltmp:" $mysql_conf;
                # start the mysql service
                /scripts/restartsrv_mysql 1>/dev/null 2>&1
                # generate the MySQL service temporary files disk usage recommendation
                part_du_data[recommendations]="$(tmp_dir_filled_with_mysql_tmp_files "$mysql_conf" "$mysqltmp" |
                        json_escape);"

                # Check if the auto fix was successful
                local tmp_m_blocks_used=$(df -B M /tmp | tail -n +2 | awk '{print $3}' | grep -oP '^[0-9]+');
                if [[ $tmp_m_blocks_used -lt $(( $tmp_m_block_capacity / 2 )) ]]; then
                        part_du_data[auto_fix_status]=$AUTO_FIX_SUCCESS;
                        part_du_data[force_report]="true";
                else
                        part_du_data[auto_fix_status]=$AUTO_FIX_FAIL;
                        part_du_data[force_report]="false";
                fi;
        fi;
}

tmp_partition_check(){
        local root_disk_usage_percentage=$(df / | tail -n +2 | awk '{print $5}' | grep -oE ^[0-9]+);
        local root_inodes_usage_percentage=$(df -i / | tail -n +2 | awk '{print $5}' | grep -oE ^[0-9]+);
        local tmp_inodes_usage_percentage=$inodes_usage_percentage;
        local dirs_to_check="/tmp";
        local fsize="1M";
        local big_files_raw=$(find $dirs_to_check -type f -size +${fsize} 2>/dev/null |
                grep -v -e "/virtfs/" -e "/.cpcpan/" | file_du | sort -rn |
                awk '{printf "%7dM\t%s\n",$1/1024/1024,$NF}' | head -10);
        [[ -n $big_files_raw ]] && part_du_data[big_files_${part}]=$(base64 <<< "$big_files_raw" |
                sed -z "s/\n/\\\n/g");

        inodes_check "$dirs_to_check";

        # File types and data extra context
        local file_type_raw=$(awk 'NR < 4 {print $2}' <<< "$big_files_raw" |
              file_type_check | base64 | sed -z "s/\n/\\\n/g");
        part_du_data[file_type]=$(json_escape <<< "$file_type_raw");

        # Show the first 256 bytes of top 3 larges files, but only if a file is a text file
        file_data=$(grep "ASCII text" <<< "$file_type_raw" | cut -d: -f1 | xargs -r head -c256 |
                json_escape);

        [[ -n $file_data ]] && part_du_data[file_data]="$file_data";

        # Check for the / partition for available disk space and inodes before attempting the mysql tmp files fix
        if [[ $root_disk_usage_percentage -lt $threshold_warning && $root_inodes_usage_percentage -lt $threshold_warning
                && $tmp_inodes_usage_percentage -lt $threshold_warning ]]; then
                mysql_tmp_files;
        else
          part_du_data[recommendations]="$(tmp_usage_general_recommendations)\n"
        fi;
}

inodes_check() {
        local check_dirs="$1";

        [[ $(df -i $part | tail -n +2 | awk '{print $5}' | tr -d '%') -ge $threshold_warning ]] && {
                part_du_data[df_inodes_${part}]="$(df -i $part | json_escape)";
                part_du_data[inodes_${part}]=$(du --inodes $check_dirs | sort -rn |
                        head -20 | sed "s/\s/\t/" | json_escape);
        };
}

backup_partition_du(){
        declare -A backup_part_du_data;
        backup_part_du_data[auto_fix_status]=$AUTO_FIX_NOPE;
        local threshold_critical=95;
        local threshold_warning=90;
        local fsize="500M";
        local part="/backup";
        local dirs_to_check="$part";
        local disk_usage_percentage=$(df $part | tail -n +2 | awk '{print $5}' | grep -oE ^[0-9]+);
        local inodes_usage_percentage=$(df -i $part | tail -n +2 | awk '{print $5}' | grep -oE ^[0-9]+);
        local recommendations
        local storage_size

        if [[ ! $(grep $part /proc/mounts) ]]; then
                backup_part_du_data[service_status]=$SERVICE_DOWN;
                backup_part_du_data[df_data_${part}]=$NO_BACKUP_MOUNT;
                unset backup_part_du_data[auto_fix_status];
        elif [[ $disk_usage_percentage -ge $threshold_warning ||
                $inodes_usage_percentage -ge $threshold_warning ]]; then

                if [[ $disk_usage_percentage -ge $threshold_critical ||
                        $inodes_usage_percentage -ge $threshold_critical ]]; then

                        backup_part_du_data[service_status]=$SERVICE_DOWN;
                else
                        backup_part_du_data[service_status]=$SERVICE_WARNING;
                fi;

                backup_part_du_data[df_data_${part}]=$(df -h $part | json_escape);

                local big_files=$(find $dirs_to_check -type f -size +${fsize} 2>/dev/null |
                        grep -v -e "/virtfs/" -e "/.cpcpan/" | file_du | sort -rn |
                        awk '{printf "%7dM\t%s\n",$1/1024/1024,$NF}' | base64 | sed -z "s/\n/\\\n/g");

                backup_part_du_data[big_dirs_${part}]=$(big_dirs "$dirs_to_check" | base64 | sed -z "s/\n/\\\n/g");

                [[ -n $big_files ]] && backup_part_du_data[big_files_${part}]="$big_files";

                [[ $inodes_usage_percentage -ge $threshold_warning ]] && {
                        backup_part_du_data[df_inodes_${part}]=$(df -i $part | json_escape);
                        inodes_check "$part";
                };
        else
                backup_part_du_data[service_status]=$SERVICE_ACTIVE;
        fi;

        if [[ ${backup_part_du_data[service_status]} == "$SERVICE_DOWN" || ${backup_part_du_data[service_status]} == "$SERVICE_WARNING" ]]; then
          general_recommendations="$(general_du_recommendations "$part")\n"
          if [ -n "${backup_part_du_data[recommendations]}" ]; then
            backup_part_du_data[recommendations]="${backup_part_du_data[recommendations]}\n$general_recommendations"
          else
            backup_part_du_data[recommendations]="$general_recommendations"
          fi

          if [[ $IS_VPS == 'true' ]] ; then
            if (( VIRT_CORE_COUNT == VPS_PULSAR_CPU || VIRT_CORE_COUNT == VPS_QUASAR_CPU )) ; then
              recommendations="$(backup_du_pulsar_quasar_recommendations)"
            elif (( VIRT_CORE_COUNT == VPS_MAGNETAR_CPU )) ; then
              recommendations="$(backup_du_magnetar_dedic_recommendations)"
            fi
          else
            storage_size="$(df -h "$part" | tail -n +2 | awk '{print $2}' | grep -oE "^[0-9]+")"
            (( storage_size != 500 )) && recommendations="$(backup_du_magnetar_dedic_recommendations)"
          fi
          backup_part_du_data[recommendations]="${backup_part_du_data[recommendations]}\n$recommendations"
        fi

        bash_arr_to_json backup_part_du_data ${!backup_part_du_data[@]} > "$BACKUP_CHECK_DATA";
}

root_partition_du(){
        declare -A part_du_data;
        declare -A part_critical_or_warning;
        local dirs_to_check="$(root_dirs_to_check)";
        local root_check_data="$ROOT_CHECK_DATA";
        local partition_to_check="$(root_disk_partitions_to_check)";

        local threshold_critical=95;
        local threshold_warning=90;
        local fsize="500M";
        local recommendations;
        part_du_data[auto_fix_status]=$AUTO_FIX_NOPE;

        for part in $partition_to_check; do
                local disk_usage_percentage=$(df $part | tail -n +2 | awk '{print $5}' | grep -oE ^[0-9]+);
                local inodes_usage_percentage=$(df -i $part | tail -n +2 | awk '{print $5}' | grep -oE ^[0-9]+);
                # Check partition only if inodes or disk usage is greater or equal to the warning threshold (90%)
                if [[ $disk_usage_percentage -ge $threshold_warning ||
                        $inodes_usage_percentage -ge $threshold_warning ]]; then
                        # When checking the / partition, the /tmp directory is treated as a separate partition,
                        # while both partitions share the same alert.
                        # The service status will be set to warning/critical
                        # if either of the partitions satisfies the warning/critical conditions.
                        if [[ ${part_du_data[service_status]} != $SERVICE_DOWN ]]; then
                                if [[ $disk_usage_percentage -ge $threshold_critical ||
                                        $inodes_usage_percentage -ge $threshold_critical ]]; then
                                        part_du_data[service_status]=$SERVICE_DOWN;
                                else
                                        part_du_data[service_status]=$SERVICE_WARNING;
                                fi;
                        fi;

                        part_critical_or_warning[$part]=1;

                        [[ $(df -h $part | tail -n +2 | awk '{print $5}' | tr -d '%') -ge $threshold_warning ]] &&
                                part_du_data[df_data_${part}]=$(df -h $part | json_escape);

                        if [[ $part == "/tmp" ]]; then
                                tmp_partition_check;
                        else
                                local big_files=$(find $dirs_to_check -type f -size +${fsize} 2>/dev/null |
                                        grep -v -e "/virtfs/" -e "/.cpcpan/" | file_du | sort -rn |
                                        awk '{printf "%7dM\t%s\n",$1/1024/1024,$NF}' | base64 |
                                        sed -z "s/\n/\\\n/g");
                                [[ -n $big_files ]] && part_du_data[big_files_${part}]="$big_files";

                                part_du_data[big_dirs_${part}]=$(big_dirs "$dirs_to_check" | base64 |
                                        sed -z "s/\n/\\\n/g");

                                [[ $inodes_usage_percentage -ge $threshold_warning ]] && {
                                        part_du_data[df_inodes_${part}]=$(df -i $part | json_escape);
                                        inodes_check "$dirs_to_check";
                                };
                        fi;

                        if [[ $part == "/" ]]; then
                                extra_root_disk_checks;
                        fi;

                # Check if either the / or /tmp partition has already set the service status to warning or critical,
                # if not, then the / partition service status, which includes the /tmp partition, is set to "ok".
                elif [[ -z ${part_du_data[service_status]} ]]; then
                        part_du_data[service_status]=$SERVICE_ACTIVE;
                fi;
        done;

        fully_managed_root_tmp_manual_review;

        if [[ ${part_critical_or_warning[/]} ]] ; then
          general_recommendations="$(general_du_recommendations "/")\n"
          if [ -n "${part_du_data[recommendations]}" ]; then
            part_du_data[recommendations]="${part_du_data[recommendations]}\n$general_recommendations"
          else
            part_du_data[recommendations]="$general_recommendations"
          fi

          if [[ $IS_VPS == 'true' ]] ; then
            if (( VIRT_CORE_COUNT == VPS_PULSAR_CPU )) ; then
               recommendations="$(vps_pulsar_du_recommendations)"
            elif (( VIRT_CORE_COUNT == VPS_QUASAR_CPU )) ; then
               recommendations="$(vps_quasar_du_recommendations)"
            elif (( VIRT_CORE_COUNT == VPS_MAGNETAR_CPU )) ; then
               recommendations="$(vps_magnetar_or_dedic_du_recommendations)"
            fi
          else
            recommendations="$(vps_magnetar_or_dedic_du_recommendations)"
          fi
        part_du_data[recommendations]="${part_du_data[recommendations]}\n$recommendations"
      fi

        bash_arr_to_json part_du_data ${!part_du_data[@]} > "$root_check_data";
}

root_dirs_to_check(){
        local dir_check_list="/home /var";
        [[ ! $(grep backup /etc/fstab) ]] && dir_check_list+=" /backup";
        [[ ! $(grep tmp /etc/fstab) ]] && dir_check_list+=" /tmp";
        echo "$dir_check_list";
}

root_disk_partitions_to_check(){
        local data_partitions=$(df | grep -P "^/dev/(([a-z]d[a-z][0-9]+)|(md[0-9]+)|(dm\-?[0-9]+))" | grep -v -e "/boot" -e "/backup" | rev |
                awk '{print $1}' | rev);

        [[ $(grep tmp /etc/fstab) ]] && data_partitions+=" /tmp";
        echo "$data_partitions";
}

disk_backup_check(){
        run_check_in_background "backup_partition_du" "$BACKUP_CHECK_DATA" "$BACKUP_CHECK_ERRORS"\
                "${CHECK_LOCKS_DIR}/${FUNCNAME}.lock";
}

disk_root_check(){
        run_check_in_background "root_partition_du" "$ROOT_CHECK_DATA" "$ROOT_CHECK_ERRORS"\
                "${CHECK_LOCKS_DIR}/${FUNCNAME}.lock";
}

big_dirs(){
        local depth;
        for dir in $@; do
                if [[ $dir =~ backup ]]; then
                        depth=6;
                else
                        depth=4;
                fi;

                du -b -d${depth} --threshold=1G --exclude=virtfs $dir 2>/dev/null | sort -rn |
                        awk '{printf "%7dG\t%s\n",$1/1024/1024/1024,$NF}';
                echo;
        done | sed -zE "s/\n{3}/\n\n/g";
}