nginx采用的也是大部分http服务器的作法,就是master,worker模型,一个master进程管理站个或者多个worker进程,基本的事件处理都是放在woker中,master负责一些全局初始化,以及对worker的管理。
在nginx中master和worker的通讯是经过socketpair来实现的,每次fork完一个子进程以后,将这个子进程的socketpaire句柄传递给前面已经存在的子进程,这样子进程之间也就能够通讯了。
nginx中fork子进程是在ngx_spawn_process中进行的:
第一个参数是全局的配置,第二个参数是子进程须要执行的函数,第三个参数是proc的参数。第四个类型。 nginx
- ngx_pid_t
- ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,
- char *name, ngx_int_t respawn)
这个函数主要的任务就是:
1 有一个ngx_processes全局数组,包含了全部的存货的子进程,这里会fork出来的子进程放入到相应的位置。并设置这个进程的相关属性。
2 建立socketpair,并设置相关属性。
3 在子进程中执行传递进来的函数。
在看详细代码以前,咱们先来看几个主要的数据结构:
首先是进程结构,这个结构体表示了一个进程。包含了它的id状态,channel等等。 数组
- typedef struct {
- ngx_pid_t pid;
- int status;
- ngx_socket_t channel[2];
-
- ngx_spawn_proc_pt proc;
- void *data;
- char *name;
- unsigned respawn:1;
- unsigned just_respawn:1;
- unsigned detached:1;
- unsigned exiting:1;
- unsigned exited:1;
- } ngx_process_t;
下面咱们来看详细的代码。
先来看第一部分:
服务器
-
- ngx_process_t ngx_processes[NGX_MAX_PROCESSES];
- ...................................
- u_long on;
- ngx_pid_t pid;
- ngx_int_t s;
- if (respawn >= 0) {
- s = respawn;
-
- } else {
- for (s = 0; s < ngx_last_process; s++) {
- if (ngx_processes[s].pid == -1) {
- break;
- }
- }
- if (s == NGX_MAX_PROCESSES) {
- ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
- "no more than %d processes can be spawned",
- NGX_MAX_PROCESSES);
- return NGX_INVALID_PID;
- }
- }
接下来新建一对socketpair句柄,而后初始化相关属性。
数据结构
- if (respawn != NGX_PROCESS_DETACHED) {
-
-
- if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1)
- {
- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
- "socketpair() failed while spawning \"%s\"", name);
- return NGX_INVALID_PID;
- }
- 。。。。。。。。。。。。。。。。。。。。。。。。。。。。
- if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) {
- ........................................................
- }
-
- if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) {
- ........................................
- }
-
- on = 1;
- if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) {
- .................................................
- }
- if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) {
-
- ..............................................
- }
- if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) {................................................
- }
-
- if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) {
- 。。。。。。。。。。。。。。。。。。。。。。。。。。
- }
- ngx_channel = ngx_processes[s].channel[1];
-
- } else {
- ngx_processes[s].channel[0] = -1;
- ngx_processes[s].channel[1] = -1;
- }
接下来就是fork子进程,并设置进程相关参数。 异步
- ngx_process_slot = s;
-
-
- pid = fork();
-
- switch (pid) {
-
- case -1:
- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
- "fork() failed while spawning \"%s\"", name);
- ngx_close_channel(ngx_processes[s].channel, cycle->log);
- return NGX_INVALID_PID;
-
- case 0
- ngx_pid = ngx_getpid();
- proc(cycle, data);
- break;
-
- default:
- break;
- }
-
- ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid);
-
- ngx_processes[s].pid = pid;
- ngx_processes[s].exited = 0;
-
- if (respawn >= 0) {
- return pid;
- }
- ngx_processes[s].proc = proc;
- ngx_processes[s].data = data;
- ngx_processes[s].name = name;
- ngx_processes[s].exiting = 0;
-
-
- switch (respawn) {
-
- case NGX_PROCESS_RESPAWN:
- ngx_processes[s].respawn = 1;
- ngx_processes[s].just_respawn = 0;
- ngx_processes[s].detached = 0;
- break;
-
- case NGX_PROCESS_JUST_RESPAWN:
- ngx_processes[s].respawn = 1;
- ngx_processes[s].just_respawn = 1;
- ngx_processes[s].detached = 0;
- break;
-
- case NGX_PROCESS_DETACHED:
- ngx_processes[s].respawn = 0;
- ngx_processes[s].just_respawn = 0;
- ngx_processes[s].detached = 1;
- break;
- }
-
- if (s == ngx_last_process) {
- ngx_last_process++;
- }
-
- return pid;
这里有个问题,那就是后面fork的子进程如何来让前面已经fork的子进程获得本身的进程相关信息呢。在nginx中是每次新的子进程fork 完毕后,而后父进程此时将这个子进程id,以及流管道的句柄channel[0]传递给前面的子进程。这样子进程之间也能够通讯了。
先来看相关的数据结构: socket
- typedef struct {
- ngx_uint_t command;
- ngx_pid_t pid;
- ngx_int_t slot;
- ngx_fd_t fd;
- } ngx_channel_t;
接下来来看代码:
函数
- static void
- ngx_start_worker_processes(ngx_cycle_t *cycle, ngx_int_t n, ngx_int_t type)
- {
- ngx_int_t i, s;
- ngx_channel_t ch;
-
- ....................................
- ch.command = NGX_CMD_OPEN_CHANNEL;
-
- for (i = 0; i < n; i++) {
-
- cpu_affinity = ngx_get_cpu_affinity(i);
-
- ngx_spawn_process(cycle, ngx_worker_process_cycle, NULL,
- "worker process", type);
- ch.pid = ngx_processes[ngx_process_slot].pid;
- ch.slot = ngx_process_slot;
- ch.fd = ngx_processes[ngx_process_slot].channel[0];
-
- for (s = 0; s < ngx_last_process; s++) {
- if (s == ngx_process_slot
- || ngx_processes[s].pid == -1
- || ngx_processes[s].channel[0] == -1)
- {
- continue;
- }
-
- ngx_log_debug6(NGX_LOG_DEBUG_CORE, cycle->log, 0,
- "pass channel s:%d pid:%P fd:%d to s:%i pid:%P fd:%d",
- ch.slot, ch.pid, ch.fd,
- s, ngx_processes[s].pid,
- ngx_processes[s].channel[0]);
-
-
- ngx_write_channel(ngx_processes[s].channel[0],
- &ch, sizeof(ngx_channel_t), cycle->log);
- }
- }
- }
而在子进程中是如何处理的呢,子进程的管道可读事件捕捉函数是ngx_channel_handler(ngx_event_t *ev),在这个函数中,会读取mseeage,而后解析,并根据不一样的命令作不一样的处理,来看它的代码片段:
ui
-
- switch (ch.command) {
-
- case NGX_CMD_QUIT:
- ngx_quit = 1;
- break;
-
- case NGX_CMD_TERMINATE:
- ngx_terminate = 1;
- break;
-
- case NGX_CMD_REOPEN:
- ngx_reopen = 1;
- break;
-
- case NGX_CMD_OPEN_CHANNEL:
- ngx_processes[ch.slot].pid = ch.pid;
- ngx_processes[ch.slot].channel[0] = ch.fd;
- break;
-
- case NGX_CMD_CLOSE_CHANNEL:
- .....................................................
-
- if (close(ngx_processes[ch.slot].channel[0]) == -1) {
- ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno,
- "close() channel failed");
- }
-
- ngx_processes[ch.slot].channel[0] = -1;
- break;
- }
接下来详细的来看worker和master如何进行交互,以及master如何同外部进行交互(好比热代码替换,reconfig这些操做)。
在nginx中,worker和master的交互,咱们前面已经提过了,就是经过流管道以及信号,而master与外部的交互是经过信号来进行的。
在看master得主循环以前,咱们先来看信号处理和函数,在nginx中,父子进程的信号处理函数是相同的,只不过有一个变量在master和worker中赋值不一样,以此来区分。
在信号处理中,经过设置相应的标志变量,从而在主循环中,判断这些变量,从而作相应的操做。
spa
- #define NGX_SHUTDOWN_SIGNAL QUIT
- #define NGX_TERMINATE_SIGNAL TERM
- #define NGX_NOACCEPT_SIGNAL WINCH
- #define NGX_RECONFIGURE_SIGNAL HUP
-
- #if (NGX_LINUXTHREADS)
- #define NGX_REOPEN_SIGNAL INFO
- #define NGX_CHANGEBIN_SIGNAL XCPU
- #else
- #define NGX_REOPEN_SIGNAL USR1
- #define NGX_CHANGEBIN_SIGNAL USR2
- #endif
-
-
- void
- ngx_signal_handler(int signo)
- {
- char *action;
- ngx_int_t ignore;
- ngx_err_t err;
- ngx_signal_t *sig;
-
- ignore = 0;
-
- err = ngx_errno;
-
- for (sig = signals; sig->signo != 0; sig++) {
- if (sig->signo == signo) {
- break;
- }
- }
-
- ngx_time_update(0, 0);
-
- action = "";
-
- switch (ngx_process) {
- case NGX_PROCESS_MASTER:
- case NGX_PROCESS_SINGLE:
- switch (signo) {
-
- case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
- ngx_quit = 1;
- action = ", shutting down";
- break;
-
- case ngx_signal_value(NGX_TERMINATE_SIGNAL):
- case SIGINT:
- ngx_terminate = 1;
- action = ", exiting";
- break;
-
- case ngx_signal_value(NGX_NOACCEPT_SIGNAL):
- ngx_noaccept = 1;
- action = ", stop accepting connections";
- break;
-
- case ngx_signal_value(NGX_RECONFIGURE_SIGNAL):
- ngx_reconfigure = 1;
- action = ", reconfiguring";
- break;
-
- case ngx_signal_value(NGX_REOPEN_SIGNAL):
- ngx_reopen = 1;
- action = ", reopening logs";
- break;
- case ngx_signal_value(NGX_CHANGEBIN_SIGNAL):
- if (getppid() > 1 || ngx_new_binary > 0) {
-
-
- action = ", ignoring";
- ignore = 1;
- break;
- }
- ngx_change_binary = 1;
- action = ", changing binary";
- break;
-
- case SIGALRM:
- break;
-
- case SIGIO:
- ngx_sigio = 1;
- break;
-
- case SIGCHLD:
- ngx_reap = 1;
- break;
- }
-
- break;
- case NGX_PROCESS_WORKER:
- switch (signo) {
-
- case ngx_signal_value(NGX_NOACCEPT_SIGNAL):
- ngx_debug_quit = 1;
- case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
- ngx_quit = 1;
- action = ", shutting down";
- break;
-
- case ngx_signal_value(NGX_TERMINATE_SIGNAL):
- case SIGINT:
- ngx_terminate = 1;
- action = ", exiting";
- break;
-
- case ngx_signal_value(NGX_REOPEN_SIGNAL):
- ngx_reopen = 1;
- action = ", reopening logs";
- break;
-
- ...............................................
- }
-
- break;
- }
-
- ................................................
-
- if (signo == SIGCHLD) {
- ngx_process_get_status();
- }
-
- ngx_set_errno(err);
- }
先来看master的主循环,处理其实很简单,就是在循环过程当中判断相应的条件,而后进入相应的处理。这里的相关标志位基本都是在上面的信号处理函数中赋值的。:
debug
- for ( ;; ) {
- if (delay) {
- delay *= 2;
- ..............................................
-
- itv.it_interval.tv_sec = 0;
- itv.it_interval.tv_usec = 0;
- itv.it_value.tv_sec = delay / 1000;
- itv.it_value.tv_usec = (delay % 1000 ) * 1000;
- if (setitimer(ITIMER_REAL, &itv, NULL) == -1) {
- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
- "setitimer() failed");
- }
- }
-
- sigsuspend(&set);
-
- ngx_time_update(0, 0);
-
- ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "wake up");
-
- if (ngx_reap) {
- ngx_reap = 0;
- ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "reap children");
- live = ngx_reap_children(cycle);
- }
-
- if (!live && (ngx_terminate || ngx_quit)) {
- ngx_master_process_exit(cycle);
- }
- if (ngx_terminate) {
- if (delay == 0) {
- delay = 50;
- }
-
- if (delay > 1000) {
- ngx_signal_worker_processes(cycle, SIGKILL);
- } else {
- ngx_signal_worker_processes(cycle,
- ngx_signal_value(NGX_TERMINATE_SIGNAL));
- }
-
- continue;
- }
-
- if (ngx_quit) {
- ngx_signal_worker_processes(cycle,
- ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
-
- ls = cycle->listening.elts;
- for (n = 0; n < cycle->listening.nelts; n++) {
- if (ngx_close_socket(ls[n].fd) == -1) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
- ngx_close_socket_n " %V failed",
- &ls[n].addr_text);
- }
- }
- cycle->listening.nelts = 0;
-
- continue;
- }
-
- if (ngx_reconfigure) {
- ngx_reconfigure = 0;
- if (ngx_new_binary) {
- ngx_start_worker_processes(cycle, ccf->worker_processes,
- NGX_PROCESS_RESPAWN);
- ngx_start_cache_manager_process(cycle, NGX_PROCESS_RESPAWN);
- ngx_noaccepting = 0;
-
- continue;
- }
-
- ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reconfiguring");
- cycle = ngx_init_cycle(cycle);
- if (cycle == NULL) {
- cycle = (ngx_cycle_t *) ngx_cycle;
- continue;
- }
-
- ngx_cycle = cycle;
- ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,
- ngx_core_module);
- ngx_start_worker_processes(cycle, ccf->worker_processes,
- NGX_PROCESS_JUST_RESPAWN);
- ngx_start_cache_manager_process(cycle, NGX_PROCESS_JUST_RESPAWN);
- live = 1;
- ngx_signal_worker_processes(cycle,
- ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
- }
- if (ngx_restart) {
- ngx_restart = 0;
- ngx_start_worker_processes(cycle, ccf->worker_processes,
- NGX_PROCESS_RESPAWN);
- ngx_start_cache_manager_process(cycle, NGX_PROCESS_RESPAWN);
- live = 1;
- }
- if (ngx_reopen) {
- ngx_reopen = 0;
- ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reopening logs");
- ngx_reopen_files(cycle, ccf->user);
- ngx_signal_worker_processes(cycle,
- ngx_signal_value(NGX_REOPEN_SIGNAL));
- }
-
- if (ngx_change_binary) {
- ngx_change_binary = 0;
- ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "changing binary");
- ngx_new_binary = ngx_exec_new_binary(cycle, ngx_argv);
- }
- if (ngx_noaccept) {
- ngx_noaccept = 0;
- ngx_noaccepting = 1;
- ngx_signal_worker_processes(cycle,
- ngx_signal_value(NGX_SHUTDOWN_SIGNAL));
- }
- }
- }
而后来看worker的主循环,worker的比较简单。逻辑和master的很类似:
- for ( ;; ) {
- if (ngx_exiting) {
-
- c = cycle->connections;
- .............................................
- if (ngx_event_timer_rbtree.root == ngx_event_timer_rbtree.sentinel)
- {
- ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "exiting");
-
- ngx_worker_process_exit(cycle);
- }
- }
-
- ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "worker cycle");
-
- ngx_process_events_and_timers(cycle);
-
- if (ngx_terminate) {
- ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "exiting");
-
- ngx_worker_process_exit(cycle);
- }
-
- if (ngx_quit) {
- ngx_quit = 0;
- ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0,
- "gracefully shutting down");
- ngx_setproctitle("worker process is shutting down");
-
- if (!ngx_exiting) {
- ngx_close_listening_sockets(cycle);
- ngx_exiting = 1;
- }
- }
-
- if (ngx_reopen) {
- ngx_reopen = 0;
- ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reopening logs");
- ngx_reopen_files(cycle, -1);
- }
- }