1、问题描述
140.207.202.187 - - [18/May/2016:10:30:58 +0800] "POST/v3/violations HTTP/1.1" 499 0 "-" "-"
42.236.10.71 - - [18/May/2016:10:30:59 +0800] "POST /v3/violationsHTTP/1.1" 499 0 "-" "-"2、问题分析
google
499 / ClientClosed Request
An Nginx HTTP server extension. This codeis introduced to log the case when the connection is closed by client whileHTTP server is processing its request, making server unable to send the HTTP header back维基百科
499Client Closed Request (Nginx)
Used in Nginx logs to indicate when the connection has been closed by client while the server is still processing itsrequest, making server unable to send a status code backnginx源码
./src/http/ngx_http_core_module.c: if (status == NGX_ERROR || status == 499) {
./src/http/ngx_http_request.h:#define NGX_HTTP_CLIENT_CLOSED_REQUEST 499
./src/http/ngx_http_special_response.c: ngx_null_string, /* 499, client has closed connection */这是nginx定义的一个状态码，用于表示这样的错误：服务器返回http头之前，客户端就提前关闭了http连接，这很有可能是因为服务器端处理的时间过长，客户端“不耐烦”了。
进一步查找
查找“NGX_HTTP_CLIENT_CLOSED_REQUEST”
./src/http/ngx_http_upstream.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_upstream.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_upstream.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_upstream.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_upstream.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_upstream.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_request.c: || rc == NGX_HTTP_CLIENT_CLOSED_REQUEST
./src/http/ngx_http_spdy.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_spdy.c: ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_spdy.c: ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_spdy.c: ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/modules/tfs/ngx_http_tfs_server_handler.c: return NGX_HTTP_CLIENT_CLOSED_REQUEST;
./src/http/modules/tfs/ngx_http_tfs.c: if (rc == NGX_HTTP_CLIENT_CLOSED_REQUEST
./src/http/modules/tfs/ngx_http_tfs.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/modules/lua/ngx_http_lua_socket_tcp.c: if (rc == NGX_HTTP_CLIENT_CLOSED_REQUEST) {
./src/http/modules/lua/ngx_http_lua_control.c: && rc != NGX_HTTP_CLIENT_CLOSED_REQUEST)
./src/http/modules/lua/ngx_http_lua_control.c: && rc != NGX_HTTP_CLIENT_CLOSED_REQUEST
./src/http/modules/lua/ngx_http_lua_subrequest.c: && rc != NGX_HTTP_CLIENT_CLOSED_REQUEST))
./src/http/modules/lua/ngx_http_lua_util.c: return NGX_HTTP_CLIENT_CLOSED_REQUEST;
./src/http/modules/lua/ngx_http_lua_util.c: return NGX_HTTP_CLIENT_CLOSED_REQUEST;
./src/http/modules/lua/ngx_http_lua_util.c: return NGX_HTTP_CLIENT_CLOSED_REQUEST;
./src/http/ngx_http_spdy_v3.c: NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_spdy_v3.c: ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_spdy_v3.c: ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_spdy_v3.c: ngx_http_spdy_finalize_connection(sc, NGX_HTTP_CLIENT_CLOSED_REQUEST);
./src/http/ngx_http_request.h:#define NGX_HTTP_CLIENT_CLOSED_REQUEST 499第一处显示
#if (NGX_HAVE_KQUEUE)
if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) {
if (!ev->pending_eof) {
return;
}
ev->eof = 1;
c->error = 1;
if (ev->kq_errno) {
ev->error = 1;
}
if (!u->cacheable && u->peer.connection) {
ngx_log_error(NGX_LOG_INFO, ev->log, ev->kq_errno,
"kevent() reported that client prematurely closed "
"connection, so upstream connection is closed too");
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_CLIENT_CLOSED_REQUEST);
return;
}
ngx_log_error(NGX_LOG_INFO, ev->log, ev->kq_errno,
"kevent() reported that client prematurely closed "
"connection");
if (u->peer.connection == NULL) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_CLIENT_CLOSED_REQUEST);
}
return;
}KQUEUE 这个是在unix下使用的网络模型，这里可以略过
第二处显示
static void
ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
ngx_event_t *ev)
{
int n;
char buf[1];
ngx_err_t err;
ngx_int_t event;
ngx_connection_t *c;
ngx_http_upstream_t *u;
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, ev->log, 0,
"http upstream check client, write event:%d, \"%V\"",
ev->write, &r->uri);
c = r->connection;
u = r->upstream;
if (c->error) {
if ((ngx_event_flags & NGX_USE_LEVEL_EVENT) && ev->active) {
event = ev->write ? NGX_WRITE_EVENT : NGX_READ_EVENT;
if (ngx_del_event(ev, event, 0) != NGX_OK) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
}
if (!u->cacheable) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_CLIENT_CLOSED_REQUEST);
}
return;
}第三处显示
void
ngx_http_finalize_request(ngx_http_request_t *r, ngx_int_t rc)
{
ngx_connection_t *c;
ngx_http_request_t *pr;
ngx_http_core_loc_conf_t *clcf;
c = r->connection;
ngx_log_debug5(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http finalize request: %d, \"%V?%V\" a:%d, c:%d",
rc, &r->uri, &r->args, r == c->data, r->main->count);
if (rc == NGX_DONE) {
ngx_http_finalize_connection(r);
return;
}
if (rc == NGX_OK && r->filter_finalize) {
c->error = 1;
}
if (rc == NGX_DECLINED) {
r->content_handler = NULL;
r->write_event_handler = ngx_http_core_run_phases;
ngx_http_core_run_phases(r);
return;
}
if (r != r->main && r->post_subrequest) {
rc = r->post_subrequest->handler(r, r->post_subrequest->data, rc);
}
if (rc == NGX_ERROR
|| rc == NGX_HTTP_REQUEST_TIME_OUT
|| rc == NGX_HTTP_CLIENT_CLOSED_REQUEST
|| c->error)
{
if (ngx_http_post_action(r) == NGX_OK) {
return;
}
if (r->main->blocked) {
r->write_event_handler = ngx_http_request_finalizer;
}
ngx_http_terminate_request(r, rc);
return;
}在收到读写事件处理之前时连接不可用
1、在一个upstream出错，执行next_upstream时也会判断连接是否可用，不可用则返回499
2、server处理请求未结束，而client提前关闭了连接
3、故障排除
nginx配置中加上
proxy_ignore_client_abort on; #表示代理服务端不要主要主动关闭客户端连接。 默认 proxy_ignore_client_abort 是关闭的，
关闭时：如果客户端端主动关闭请求或者客户端网络断掉， Nginx 会记录 499，同时 request_time 是「后端已经处理」的时间，而upstream_response_time 为“-“ (已验证)。
开启时：如果客户端端主动关闭请求或者客户端网络断掉，Nginx 会等待后端处理完(或者超时)，然后记录「后端的返回信息」到日志。所以，如果后端返回 200，就记录 200 ；如果后端放回 5XX ，那么就记录 5XX 。
如果超时(默认60s，可以用 proxy_read_timeout 设置)，Nginx 会主动断开连接，记录 504
注：只在做反向代理的时候加入，作为其他服务器的时候，关闭为好，默认设置是关闭的！
WEB架构
安全监控体系
集群架构
