When you do a reverse proxy in which Nginx/varnish/ApacheBooster will be the one who listen the traffic and forward anything needs to be process through PHP to Apache. With this, Apache will always receive an IP from frontend server (or I should say localhost / 127.0.0.1 / local IP/ server main IP), not the real IP that user uses. In order to let Apache knows what is real user IP, we need to install mod_rpaf, but the apache 2.4 is not suppoting current mod_rpaf version, so you need to apply the following patch to work with apache2.4.

--- mod_rpaf-2.0.c.org  2012-05-17 12:05:34.082130109 +0900
+++ mod_rpaf-2.0.c      2012-05-17 12:16:41.648138252 +0900
@@ -147,8 +147,8 @@
 
 static apr_status_t rpaf_cleanup(void *data) {
     rpaf_cleanup_rec *rcr = (rpaf_cleanup_rec *)data;
-    rcr->r->connection->remote_ip   = apr_pstrdup(rcr->r->connection->pool, rcr->old_ip);
-    rcr->r->connection->remote_addr->sa.sin.sin_addr.s_addr = apr_inet_addr(rcr->r->connection->remote_ip);
+    rcr->r->connection->client_ip   = apr_pstrdup(rcr->r->connection->pool, rcr->old_ip);
+    rcr->r->connection->client_addr->sa.sin.sin_addr.s_addr = apr_inet_addr(rcr->r->connection->client_ip);
     return APR_SUCCESS;
 }
 
@@ -161,7 +161,7 @@
     if (!cfg->enable)
         return DECLINED;
 
-    if (is_in_array(r->connection->remote_ip, cfg->proxy_ips) == 1) {
+    if (is_in_array(r->connection->client_ip, cfg->proxy_ips) == 1) {
         /* check if cfg->headername is set and if it is use
            that instead of X-Forwarded-For by default */
         if (cfg->headername && (fwdvalue = apr_table_get(r->headers_in, cfg->headername))) {
@@ -180,11 +180,11 @@
                 if (*fwdvalue != '\0')
                     ++fwdvalue;
             }
-            rcr->old_ip = apr_pstrdup(r->connection->pool, r->connection->remote_ip);
+            rcr->old_ip = apr_pstrdup(r->connection->pool, r->connection->client_ip);
             rcr->r = r;
             apr_pool_cleanup_register(r->pool, (void *)rcr, rpaf_cleanup, apr_pool_cleanup_null);
-            r->connection->remote_ip = apr_pstrdup(r->connection->pool, ((char **)arr->elts)[((arr->nelts)-1)]);
-            r->connection->remote_addr->sa.sin.sin_addr.s_addr = apr_inet_addr(r->connection->remote_ip);
+            r->connection->client_ip = apr_pstrdup(r->connection->pool, ((char **)arr->elts)[((arr->nelts)-1)]);
+            r->connection->client_addr->sa.sin.sin_addr.s_addr = apr_inet_addr(r->connection->client_ip);
             if (cfg->sethostname) {
                 const char *hostvalue;
                 if (hostvalue = apr_table_get(r->headers_in, "X-Forwarded-Host")) {

save the above codes in text file and apply the patch using the following command.
patch -p1 < patch_mod_rpaf