Nginx 实现冷热文件分区读取

  • 10.2.2.2 为热文件服务器,存储1年内上传的文件
  • 10.3.3.3 为冷文件服务器,存储上传超过1年的文件
  • 冷热文件服务器不互通,但都可以响应静态文件网关的请求
upstream host_s1 {
    server 10.2.2.2:443;
    server 10.3.3.3:443 backup;
}

server {

    listen 80;

    server_name s1.rehiy.com;

    location / {
        proxy_pass http://host_s1;
        proxy_connect_timeout 5s;
        proxy_next_upstream_tries 2;
        proxy_next_upstream_timeout 5s;
        proxy_next_upstream error timeout http_404 http_502;
    }

}

批量将discuz数据库编码从utf8转为utf8mb4

为获得emoji等特殊字符支持,需要将discuz的数据库编码转换为utf8mb4

  • 确认数据库版本支持utf8mb4utf8mb4_0900_ai_ci
  • 由于用户表中username是不重复索引,且区分大小写和声调,字符集需设置为utf8mb4_general_ci
  • 手动执行下面的SELECT CONCAT语句,然后使用执行结果替换该语句,保存为一个 sql 文件
  • 为避免 php 运行超时,请使用 cli 工具导入保存的 sql 文件
USE `discuz_databese`;

-- 更改主库字符集

ALTER DATABASE `discuz_databese` COLLATE utf8mb4_0900_ai_ci;

-- 请使用下列语句的执行结果替换该语句

SELECT CONCAT(
  'ALTER TABLE ', `TABLE_NAME`, ' CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;'
) FROM `information_schema`.`TABLES`
  WHERE `TABLE_SCHEMA` = 'discuz_databese'
    AND `TABLE_NAME` <> 'pre_common_member'
    AND `TABLE_NAME` <> 'pre_common_member_archive'
    AND `TABLE_NAME` <> 'pre_ucenter_members';

-- 单独修改用户表(`username`字符集为`utf8mb4_general_ci`)

ALTER TABLE `pre_common_member` COLLATE utf8mb4_0900_ai_ci,
  MODIFY `email` char(40) NOT NULL DEFAULT '',
  MODIFY `username` char(15) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
  MODIFY `password` char(32) NOT NULL DEFAULT '',
  MODIFY `extgroupids` char(20) NOT NULL DEFAULT '',
  MODIFY `timeoffset` char(4) NOT NULL DEFAULT '';

ALTER TABLE `pre_common_member_archive` COLLATE utf8mb4_0900_ai_ci,
  MODIFY `email` char(40) NOT NULL DEFAULT '',
  MODIFY `username` char(15) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
  MODIFY `password` char(32) NOT NULL DEFAULT '',
  MODIFY `extgroupids` char(20) NOT NULL DEFAULT '',
  MODIFY `timeoffset` char(4) NOT NULL DEFAULT '';

ALTER TABLE `pre_ucenter_members` COLLATE utf8mb4_0900_ai_ci,
  MODIFY `username` char(15) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
  MODIFY `password` char(32) NOT NULL DEFAULT '',
  MODIFY `email` char(32) NOT NULL DEFAULT '',
  MODIFY `myid` char(30) NOT NULL DEFAULT '',
  MODIFY `myidkey` char(16) NOT NULL DEFAULT '',
  MODIFY `regip` char(15) NOT NULL DEFAULT '',
  MODIFY `salt` char(6) NOT NULL,
  MODIFY `secques` char(8) NOT NULL DEFAULT '';

下面是写给强迫症患者的,你可能并不需要 !!

-- 创建临时库

CREATE DATABASE `tmp123`;

-- 从原始数据库复制表(会丢弃索引)

USE `discuz_database`;

CREATE TABLE `tmp123`.`pre_common_member` AS SELECT * FROM `pre_common_member`;
CREATE TABLE `tmp123`.`pre_common_member_archive` AS SELECT * FROM `pre_common_member_archive`;

CREATE TABLE `tmp123`.`pre_ucenter_members` AS SELECT * FROM `pre_ucenter_members`;

-- 进入临时库

USE `tmp123`;

-- 转换数据库编码和字符集(已无索引,可以转换)

ALTER TABLE `pre_common_member` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;
ALTER TABLE `pre_common_member_archive` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;

ALTER TABLE `pre_ucenter_members` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;

-- 获取重复的用户统计到临时表

CREATE TABLE `pre_ucenter_members_having_count` AS
  SELECT `username`, count(username) as `count` FROM `pre_ucenter_members`
    GROUP BY `username` HAVING `count` > 1;

-- 获取重复的用户详情到临时表

CREATE TABLE `pre_ucenter_members_having` AS
  SELECT * FROM `pre_ucenter_members` WHERE `username` IN (
      SELECT `username` FROM `pre_ucenter_members_having_count`
  );

Gitlab 保存设置项目时提示500错误

将gitlab转为docker后,发现管理后台保存配置时提示500错误,查看日志/var/log/gitlab/gitlab-rails/production.log对应的错误信息为OpenSSL::Cipher::CipherError

查阅官方资料,得知是没有导入原来的加密信息/etc/gitlab/gitlab-secrets.json导致,重置下即可。

  • 进入控制台docker exec -it docker bash
  • 输入gitlab-rails console,等待进入rails控制台
  • 输入ApplicationSetting.current.reset_runners_registration_token!,显示结果为true
  • 再次尝试保存操作,已正常

如果项目设置也不正常,需进入DB控制台清理更多的Token设置,请参阅官方文献:
https://docs.gitlab.com/ee/raketasks/backup_restore.html#when-the-secrets-file-is-lost

PostgreSQL 更改库及所有表的所有者(OWNER)

  • database, public,schema1 为你需要修改的库名
  • username 为更改后的所有者用户名
\c database;

DO $$
DECLARE
    r record;
    i int;
    v_schema text[] := '{public,schema1}';
    v_new_owner varchar := 'username';
BEGIN
    FOR r IN
        SELECT 'ALTER TABLE "' || table_schema || '"."' || table_name || '" OWNER TO ' || v_new_owner || ';' AS a FROM information_schema.tables WHERE table_schema = ANY (v_schema)
        UNION ALL
        SELECT 'ALTER TABLE "' || sequence_schema || '"."' || sequence_name || '" OWNER TO ' || v_new_owner || ';' AS a FROM information_schema.sequences WHERE sequence_schema = ANY (v_schema)
        UNION ALL
        SELECT 'ALTER TABLE "' || table_schema || '"."' || table_name || '" OWNER TO ' || v_new_owner || ';' AS a FROM information_schema.views WHERE table_schema = ANY (v_schema)
        UNION ALL
        SELECT 'ALTER FUNCTION "' || nsp.nspname || '"."' || p.proname || '"(' || pg_get_function_identity_arguments(p.oid) || ') OWNER TO ' || v_new_owner || ';' AS a FROM pg_proc p JOIN pg_namespace nsp ON p.pronamespace = nsp.oid WHERE nsp.nspname = ANY (v_schema)
        UNION ALL
        SELECT 'ALTER DATABASE "' || current_database() || '" OWNER TO ' || v_new_owner
    LOOP
        EXECUTE r.a;
    END LOOP;
    FOR i IN array_lower(v_schema, 1)..array_upper(v_schema, 1)
    LOOP
        EXECUTE 'ALTER SCHEMA "' || v_schema[i] || '" OWNER TO ' || v_new_owner;
    END LOOP;
END
$$;

自动生成百度云加速IP白名单

百度云加速的ip段非常之多,官方给了一个帖子来列出这些ip,获取起来十分不便。
倍感痛苦的我,最终还是决定写个PHP脚本自动更新Nginx的real_ip规则。其他规则也可以参考修改。

最新源码参看:https://github.com/anrip/baidu-yunjiasu-ip

<?php
$bdip = read_ip_list('https://ticket-baidu.kf5.com/posts/view/148628');
$cfip = read_ip_list('https://www.cloudflare.com/ips-v4');
$list = array_merge($bdip, $cfip);

make_nginx_real_ip_conf($list);

///////////////////////////////////////////////////////////

function make_nginx_real_ip_conf($list) {
    foreach($list as &$ip) {
        $ip = "set_real_ip_from {$ip};";
    }
    $text = implode("\n", $list);
    file_put_contents('nginx_real_ip.conf', $text);
}

function read_ip_list($site) {
    $html = file_get_contents($site);
    if(!preg_match_all('/\d+\.\d+\.\d+\.\d+\/\d+/', $html, $list)) {
        exit("读取远程数据失败: {$site}\n");
    }
    return sort_ip_list($list[0]);
}

function sort_ip_list($list) {
    $rets = array();
    foreach(array_unique($list) as $val) {
        $ip = ip2long(explode('/', $val)[0]);
        $ip = sprintf('%u', floatval($ip));
        $rets[$ip] = $val;
    }
    ksort($rets);
    return array_values($rets);
}

使用nginx-image-filter缩放或裁切图片

在编译Nginx时,记得增加http_image_filter模块,然后在server区域加入如下代码即可。
如果你懒得编译这个模块,也可以下载我编译好的Webox直接安装。

#生成缩略图(等比例)
#http://www.rehiy.com/expamle/test.jpg?r128x128
if ($args ~ ^r\d+x\d+$) {
    rewrite ^(.+)$ $1/$args? last;
}
location ~* \.(gif|png|jpg|jpeg)/r(\d+)x(\d+)$ {
    rewrite ^(.+)/r(\d+)x(\d+)$ $1 break;
    image_filter resize  $2 $3;
    image_filter_buffer  10M;
}
#生成裁切图片(固定高宽)
#http://www.rehiy.com/expamle/test.jpg?c128x128
if ($args ~ ^c\d+x\d+$) {
    rewrite ^(.+)$ $1/$args? last;
}
location ~* \.(gif|png|jpg|jpeg)/c(\d+)x(\d+)$ {
    rewrite ^(.+)/c(\d+)x(\d+)$ $1 break;
    image_filter crop  $2 $3;
    image_filter_buffer  10M;
}

命令行登录mysql提示Segmentation fault

自己编译的mysql,在使用命令行登录时,提示该错误,经过查阅资料,解决方案如下:

1、在源代码目录中,编辑文件 cmd-line-utils/libedit/terminal.c

2、terminal_set 方法中:
a、注释 char buf[TC_BUFSIZE];
b、把 area = buf; 改为 area = NULL;

3、重新编译

Nginx安装TWiki

一、安装必备的软件包

sudo locale-gen zh_CN
sudo locale-gen zh_CN.UTF-8
sudo dpkg-reconfigure locales
sudo perl -MCPAN -e 'install HTML::Template'
sudo apt-get install nginx fcgiwrap rcs

二、配置Nginx站点

server {

    listen       80;
    server_name  wiki.rehiy.com;

    root /srv/webroot/wiki/;
    index index.html;

    rewrite ^/$          /bin/view;
    rewrite ^/([A-Z].*)  /bin/view/$1;
    rewrite ^/edit/(.*)  /bin/edit/$1;

    location / {
        deny all;
    }

    location ~ ^/pub/ {
        allow all;
    }

    location ~ ^/bin/ {
        allow all;
        fastcgi_pass   unix:/var/run/fcgiwrap.socket;
        fastcgi_split_path_info  ^(/bin/[^/]+)(/.*)$;
        include        fastcgi_params;
        fastcgi_param  PATH_INFO        $fastcgi_path_info;
        fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
    }

}

三、配置TWiki站点

http://wiki.rehiy.com/bin/configure
此处比较重要,可能会提示一些缺少的软件包等,务必要解决;
另外,语言等也是在此处设置,无需像网上所讲的需要从源代码修改配置。

解决谷歌字体库导致页面加载慢

由于众所周知的原因,谷歌WEB字体库在中国是无法正常使用的,而很多国外的著名开源程序都使用了该库,比如wordpress、opencart等等。

网上也有很多避开这个问题的方法,下面我给出2种基于Nginx的方法。

一、使用360提供的中国镜像:

subs_filter fonts.googleapis.com fonts.useso.com;

二、基于Nginx的本地化的方法:

subs_filter (https?:)?//fonts.gstatic.com/ / r;
subs_filter (https?:)?//fonts.googleapis.com/ / r;

location ~ ^/css$ {
    proxy_set_header Accept-Encoding '';
    proxy_set_header Host fonts.googleapis.com;
    proxy_pass http://fonts.useso.com;
}

location ~ ^/s/opensans/ {
    proxy_set_header Accept-Encoding '';
    proxy_set_header Host fonts.gstatic.com;
    proxy_pass http://fonts.gstatic.com;
}

提示:
1.以上配置并未启用Nginx缓存,若有需要请自行添加
2.此配置依赖Nginx第三方扩展模块subs_filter

设置CORS实现跨域请求

一、使用php代码实现

#
# CORS config for php
# Code by anrip[wang@rehiy.com]
#

function make_cors($origin = '*') {

    $request_method = $_SERVER['REQUEST_METHOD'];

    if ($request_method === 'OPTIONS') {

        header('Access-Control-Allow-Origin:'.$origin);
        header('Access-Control-Allow-Credentials:true');
        header('Access-Control-Allow-Methods:GET, POST, OPTIONS');

        header('Access-Control-Max-Age:1728000');
        header('Content-Type:text/plain charset=UTF-8');
        header('Content-Length: 0',true);

        header('status: 204');
        header('HTTP/1.0 204 No Content');

    }

    if ($request_method === 'POST') {

        header('Access-Control-Allow-Origin:'.$origin);
        header('Access-Control-Allow-Credentials:true');
        header('Access-Control-Allow-Methods:GET, POST, OPTIONS');

    }

    if ($request_method === 'GET') {

        header('Access-Control-Allow-Origin:'.$origin);
        header('Access-Control-Allow-Credentials:true');
        header('Access-Control-Allow-Methods:GET, POST, OPTIONS');

    }

}

二、使用nginx配置实现

#
# CORS config for nginx
# Code by anrip[wang@rehiy.com]
#

location / {

    set $origin '*';

    if ($request_method = 'OPTIONS') {

        add_header 'Access-Control-Allow-Origin' $origin;

        #
        # Om nom nom cookies
        #
        add_header 'Access-Control-Allow-Credentials' 'true';
        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';

        #
        # Custom headers and headers various browsers *should* be OK with but aren't
        #
        add_header 'Access-Control-Allow-Headers' 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';

        #
        # Tell client that this pre-flight info is valid for 20 days
        #
        add_header 'Access-Control-Max-Age' 1728000;
        add_header 'Content-Type' 'text/plain charset=UTF-8';
        add_header 'Content-Length' 0;

        return 204;

    }

    if ($request_method = 'POST') {

        add_header 'Access-Control-Allow-Origin' $origin;
        add_header 'Access-Control-Allow-Credentials' 'true';
        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
        add_header 'Access-Control-Allow-Headers' 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';

    }

    if ($request_method = 'GET') {

        add_header 'Access-Control-Allow-Origin' $origin;
        add_header 'Access-Control-Allow-Credentials' 'true';
        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
        add_header 'Access-Control-Allow-Headers' 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';

    }

}