[squid-users] High tcp_hit times

My cache performance is acting strange; I'm getting extremely high
tcp_hit times for cached objects:

1212787643.465 50343 10.2.7.22 TCP_HIT/200 19290 GET http://cache-int/....
1212787737.740 15212 10.2.7.25 TCP_HIT/200 11511 GET http://cache-int/....

Those high times comes in bursts. Eg: bunch of high response time
will come followed by a normal response times. Normal response times
are sub 100ms

1212787849.726 20 10.2.7.62 TCP_HIT/200 24149 GET http://cache-int/
1212787850.352 30 10.2.7.62 TCP_HIT/200 25469 GET http://cache-int/g

The cache server is part of a reverse proxy pool. We have 8 servers
and only this one is acting weird. Any ideas on how find the cause
of the issue?

mike

config:
6GB of RAM
64bit debian 4.0
32bit squid, 2.6stable20
3ware raid controller, sets of 2 disk mirrors

output from top:
top - 14:36:19 up 1:43, 1 user, load average: 1.27, 0.77, 2.57
Tasks: 88 total, 1 running, 87 sleeping, 0 stopped, 0 zombie
Cpu(s): 0.2%us, 0.0%sy, 0.0%ni, 99.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Mem: 6111580k total, 4968848k used, 1142732k free, 211724k buffers
Swap: 1959888k total, 116k used, 1959772k free, 1326964k cached

squid conf:

acl apache rep_header Server ^Apache
broken_vary_encoding allow apache
access_log /apps/squid/var/logs/access.log squid
acl all src 0.0.0.0/0.0.0.0
acl manager proto cache_object
acl localhost src 127.0.0.1/255.255.255.255
acl to_localhost dst 127.0.0.0/8
acl SSL_ports port 443
acl Safe_ports port 80 # http
acl Safe_ports port 21 # ftp
acl Safe_ports port 443 # https
acl Safe_ports port 70 # gopher
acl Safe_ports port 210 # wais
acl Safe_ports port 1025-65535 # unregistered ports
acl Safe_ports port 280 # http-mgmt
acl Safe_ports port 488 # gss-http
acl Safe_ports port 591 # filemaker
acl Safe_ports port 777 # multiling http
acl CONNECT method CONNECT
http_access allow manager localhost
http_access deny manager
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_reply_access allow all
icp_access allow all
coredump_dir /apps/squid/var/cache
cache_effective_user squid
url_rewrite_host_header off
http_port 80 vhost defaultsite=c2.globexplorer.com
http_port 81
refresh_pattern -i gexservlets 525948 100 1051896 override-expire ignore-reload
refresh_pattern -i cgi-bin 525948 100 1051896 override-expire ignore-reload
acl textfiles rep_mime_type -i ^text/plain$
acl htmlfiles rep_mime_type -i ^text/html$
acl jpegfiles rep_mime_type -i ^image/jpeg$
acl pngfiles rep_mime_type -i ^image/png$
acl giffiles rep_mime_type -i ^image/gif$
cache deny textfiles
cache deny htmlfiles
cache_store_log none
pid_filename /var/run/squid.pid
cache_dir aufs /mnt/cache_a/squid-cache 85332 320 256
cache_dir aufs /mnt/cache_b/squid-cache 85332 320 256
cache_dir aufs /mnt/cache_c/squid-cache 85332 320 256
cache_dir aufs /mnt/cache_d/squid-cache 85332 320 256
cache_dir aufs /mnt/cache_e/squid-cache 85332 320 256
cache_dir aufs /mnt/cache_f/squid-cache 85332 320 256

cache_mem 512 MB
minimum_object_size 1 KB
maximum_object_size_in_memory 1 MB
store_avg_object_size 20 KB
offline_mode on
icp_hit_stale on
acl acceleratedHosts dst 10.2.14.1 10.2.13.1
http_access allow acceleratedHosts
acl mynet src 10.2.7.0/24
acl mynet src 10.2.13.0/24
acl mynet src 10.2.14.0/24
acl mynet src 10.9.101.0/24
http_access allow mynet
http_access allow localhost
acl PURGE method PURGE
acl purge_group src 10.10.10.213 10.9.101.60 10.10.10.14 10.10.10.15
10.10.10.16 10.10.10.176
http_access allow PURGE purge_group
http_access allow PURGE localhost
http_access deny PURGE
http_access deny all
acl snmppublic snmp_community public
snmp_access allow snmppublic all
nonhierarchical_direct off
icp_query_timeout 7000
maximum_icp_query_timeout 10000
strip_query_terms off
log_icp_queries off
cache_peer 10.2.14.1 parent 80 0 no-query originserver
no-netdb-exchange no-digest name=mw_pool
hierarchy_stoplist cmd=info cmd=checkextent
visible_hostname cache4
unique_hostname cache4
cache_peer 10.2.7.20 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache1
cache_peer 10.2.7.21 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache2
cache_peer 10.2.7.22 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache3
cache_peer 10.2.7.24 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache5
cache_peer 10.2.7.25 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache6
cache_peer 10.2.7.62 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache7-1
cache_peer 10.2.7.63 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache7-2
cache_peer 10.2.7.64 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache8-1
cache_peer 10.2.7.65 sibling 80 3130 proxy-only no-delay allow-miss
weight=1 no-netdb-exchange no-digest name=cache8-2
Received on Fri Jun 06 2008 - 21:38:41 MDT