=Start=<\/p>\n
\u4e4b\u524d\u5c31\u505a\u8fc7\u7f51\u7ad9\u8bbf\u95ee\u65e5\u5fd7\u7684\u5206\u6790\uff0c\u5206\u522b\u7528\u7684\u662fawk\u548cPython\uff0c\u4f46\u90fd\u6bd4\u8f83\u7b80\u5355\uff0c\u53ea\u662f\u7b80\u5355\u9009\u53d6\u4e86IP\u7684\u7ef4\u5ea6\uff0c\u8fd8\u6ca1\u6709\u533a\u5206\u8bbf\u95ee\u7684\u65e5\u671f\uff0c\u8fd9\u6837\u7684\u8bdd\u5982\u679c\u67d0\u4e2aIP\u5728\u67d0\u4e00\u5929\u4e2d\u7684\u8bbf\u95ee\u6b21\u6570\u5f88\u591a\uff0c\u4f46\u5728\u5176\u5b83\u8fd9\u4e00\u6708\/\u5e74\u7ec8\u7684\u5176\u4ed6\u65f6\u5019\u8bbf\u95ee\u6b21\u6570\u4e0d\u591a\uff0c\u90a3\u5728\u5355\u7eaf\u53ea\u8003\u8651IP\u8bbf\u95ee\u6b21\u6570\u7684\u60c5\u51b5\u4e0b\u662f\u5f88\u96be\u786e\u5b9a\u8fd9\u6837\u7684IP\u7684\uff08\u800c\u8fd9\u6837\u7684IP\u53c8\u5f88\u6709\u53ef\u80fd\u662f\u722c\u866b\u6216\u662f\u653b\u51fb\u8005\u7684IP\uff09\uff0c\u56e0\u6b64\u9700\u8981\u52a0\u4e0a\u65f6\u95f4\u7ef4\u5ea6\uff0c\u4f46\u5982\u679c\u65f6\u95f4\u7ef4\u5ea6\u505a\u7684\u592a\u7ec6\uff0c\u6d88\u8017\u65f6\u95f4\u592a\u957f\uff1b\u5982\u679c\u505a\u7684\u592a\u7c97\u53c8\u4e0d\u5bb9\u6613\u53d1\u73b0\u95ee\u9898\uff1b\u6240\u4ee5\uff0c\u8fd9\u79cd\u60c5\u51b5\u4e0b\u9700\u8981\u6839\u636e\u5177\u4f53\u60c5\u51b5 & \u5b9e\u9645\u9700\u8981\u6765\u9009\u53d6\u7c92\u5ea6\u3002\u56e0\u4e3a\u6211\u8fd9\u4e3b\u8981\u662f\u5404\u4eba\u5206\u6790\u7528\uff0c\u6240\u4ee5\u53ef\u4ee5\u4f7f\u7528\u7b80\u5355\u7684\u201c\u4ee5\u5929\u4e3a\u5355\u4f4d\u201d\u6765\u5206\u6790\uff0c\u7528\u5230\u4e86Python\u548cawk\u8fdb\u884c\u65e5\u5fd7\u5904\u7406\uff0c\u5feb\u901f\u65b9\u4fbf\uff1a<\/p>\n
\u9ed8\u8ba4\u7684Nginx\u8bb0\u5f55\u65e5\u5fd7\u7684\u683c\u5f0f\u4e3a\uff08\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u4ee5\u201c\u7a7a\u683c\u201d\u4e3a\u5206\u9694\u7b26\uff0c\u4f46\u5728\u771f\u5b9e\u60c5\u51b5\u4e0b\u662f\u4e0d\u5229\u4e8e\u65e5\u5fd7\u5206\u6790\u7684\uff0c\u4f46\u8fd9\u91cc\u4e3a\u4e86\u65b9\u4fbf\u8d77\u89c1\u8fd8\u662f\u4ee5\u9ed8\u8ba4\u7684\u4e3a\u793a\u4f8b\u8fdb\u884c\u8bf4\u660e\uff09\uff1a<\/p>\n
#log format\r\nlog_format access '$remote_addr\u00a0- $remote_user\u00a0[$time_local]\u00a0\"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" $http_x_forwarded_for';<\/pre>\n\u4ece\u65e5\u5fd7\u4e2d\u53d6\u4e00\u4e9b\u6837\u672c\u8fdb\u884c\u67e5\u770b\uff1a<\/p>\n
# head -3 access.log\r\n120.204.200.21 - - [25\/Apr\/2015:00:01:04 +0800] \"GET \/blog\/ HTTP\/1.1\" 200 9082 \"-\" \"DNSPod-Monitor\/2.0\" -\r\n50.116.30.23 - - [25\/Apr\/2015:00:01:05 +0800] \"GET \/blog\/feed HTTP\/1.1\" 304 0 \"-\" \"Superfeedr bot\/2.0\u00a0http:\/\/superfeedr.com\u00a0- Make your feeds realtime: get in touch - feed-id:128926078\" -\r\n209.15.21.92 - - [25\/Apr\/2015:00:01:26 +0800] \"HEAD \/blog\/ HTTP\/1.1\" 200 0 \"-\" \"-\" -<\/pre>\n\u5148\u7528awk\u53d6\u51e0\u5217\u770b\u770b\uff1a<\/p>\n
$ cat access.log | awk '$3=\"-\" {print substr($4, 2, 11),$1}'\r\n$ cat access.log | awk '$3==\"-\" {print substr($4, 2, 11),$1}'<\/pre>\n\u7528awk\u5bf9\u65e5\u5fd7\u8fdb\u884c\u5904\u7406\uff08\u6839\u636e\u201c\u65e5\u671f+IP\u201d\u8fdb\u884c\u53bb\u91cd\uff09\uff1a<\/p>\n
$ awk '{dict[substr($4, 2, 11)\"\\t\"$1]++}; END{for(item in dict) print item\"\\t\"dict[item]}' access.log<\/pre>\n\u5bf9\u5927\u516c\u53f8\u6765\u8bf4\uff0c\u65e5\u5fd7\u662f\u4f1a\u6309\u65f6\u95f4\u6bb5\u8fdb\u884c\u5207\u5272\u7684\uff08\u8fd9\u4e2a\u51e0\u4e4e\u662f\u5fc5\u7136\u7684\uff0c\u5426\u5219\u8bbf\u95ee\u65e5\u5fd7\u8fc7\u5927\u7684\u8bdd\u5bb9\u6613\u5f15\u8d77\u5404\u79cd\u95ee\u9898\/\u9690\u60a3\uff09\uff0c\u5e76\u7528\u5904\u7406\u5927\u6570\u636e\u7684\u65b9\u6cd5\u5bf9\u65e5\u5fd7\u8fdb\u884c\u5904\u7406\uff08\u5176\u5b9e\u8fd9\u51e0\u53e5\u8bdd\u90fd\u662f\u5e9f\u8bdd\uff0c\u6211\u7684\u771f\u5b9e\u76ee\u7684\u662f\u60f3\u5f15\u51faHDFS\u548c\u7b80\u5355\u7684Streaming\uff09\u2026\u2026<\/p>\n
\u6b63\u6587\uff1a<\/h4>\n
\u5e9f\u8bdd\u4e0d\u591a\u8bf4\uff0c\u901a\u8fc7awk\u5bf9\u65e5\u5fd7\u8fdb\u884c\u5904\u7406\u4e86\u4e4b\u540e\uff0c\u8bb0\u5f55\u7684\u6587\u4ef6\u5185\u5bb9\u683c\u5f0f\u4e3a\uff1a<\/p>\n
date IP count<\/p>\n
\u7136\u540e\u901a\u8fc7\u5bf9\u5904\u7406\u540e\u7684\u6587\u4ef6\u518d\u8fdb\u884c\u5206\u6790\u4ece\u800c\u5f97\u5230\u6211\u4eec\u60f3\u8981\u7684\u5404\u79cd\u7ed3\u679c\uff1a<\/p>\n
\u8fd9\u91cc\u9700\u8981\u7528\u5230\u4e8c\u7ef4\u6570\u7ec4\u7684\u77e5\u8bc6\uff08\u5982\u679c\u9700\u8981\u589e\u52a0\u66f4\u591a\u7684\u5206\u6790\u7ef4\u5ea6\uff0c\u90a3\u5c31\u9700\u8981\u591a\u7ef4\u6570\u7ec4\u4e86\uff09\uff0c\u4f46Python\u91cc\u9762\u7684\u4e8c\u7ef4\u6570\u7ec4\u7528\u7684\u5c11\uff08\u8c8c\u4f3c\u4e3b\u8981\u662f\u505a\u79d1\u7814\u7684\u7528 NumPy \u5e93\u548c\u4e00\u4e9b\u5176\u4ed6\u7684\u4e13\u7528\u5e93\uff09\uff0c\u53bbstackoverflow\u4e0a\u641c\u4e86\u4e00\u5806\u611f\u89c9\u90a3\u79cd\u4e8c\u7ef4\u6570\u7ec4\u4e0d\u662f\u6211\u60f3\u8981\u7684\uff1a<\/p>\n
==<\/p>\n
\u4e4b\u524d\u5199\u53e6\u5916\u4e00\u4e2a\u4e1c\u897f\u7684\u65f6\u5019\u7528\u4e86\u5d4c\u5957\u7684dict\u4ee3\u66ff\u4e8c\u7ef4\u6570\u7ec4\uff08list\uff09\uff0c\u633a\u597d\u7528\u7684\uff0c\u8fd9\u6b21\u5c31\u4f9d\u846b\u82a6\u753b\u74e2\uff0c\u6574\u4e86\u4e2a\u7c7b\u4f3c\u7684\uff1a<\/p>\n
#!\/usr\/bin\/env python\r\n# coding=utf-8\r\nimport sys, re\r\n\r\ndate_ip_x = {}\r\npv_of_date = {}\r\nuv_of_date = {}\r\nfor line in sys.stdin:\r\n line = line.strip()\r\n res = re.split(\" \", line)\r\n ip = res[0].strip()\r\n date = res[3].strip()[1:12]\r\n\r\n if date in date_ip_x:\r\n if ip in date_ip_x[date]:\r\n date_ip_x[date][ip] += 1\r\n else:\r\n date_ip_x[date][ip] = 1\r\n else:\r\n date_ip_x[date] = {ip:1}\r\n\r\nfor date in date_ip_x:\r\n pv_of_date[date] = len(date_ip_x[date])\r\n#pv_of_date = {date:len(date_ip_x[date]) for date in date_ip_x}\r\n\r\nfor date in date_ip_x:\r\n uv_of_date[date] = 0\r\n for ip in date_ip_x[date]:\r\n uv_of_date[date] += date_ip_x[date][ip]\r\n\r\nprint date_ip_x\r\nprint pv_of_date\r\nprint uv_of_date\r\n\r\nwith open(\"date_ip_count.txt\", 'w') as f:\r\n for date in date_ip_x:\r\n f.write('%s\\n' % date)\r\n for ip in date_ip_x[date]:\r\n f.write('%s\\t%d\\n' % (ip, date_ip_x[date][ip]))\r\n f.write('\\n')\r\n\r\nwith open(\"date_pv_uv.txt\", 'w') as f:\r\n f.write('\\n####PV_of_date####\\n')\r\n for date in pv_of_date:\r\n f.write('%s\\t%d\\n' % (date, pv_of_date[date]))\r\n f.write('\\n####UV_of_date####\\n')\r\n for date in uv_of_date:\r\n f.write('%s\\t%d\\n' % (date, uv_of_date[date]))<\/pre>\n\u6839\u636e\u4ee5\u4e0a\u7684\u51e0\u4e2a\u6b65\u9aa4\u5904\u7406\u4e86\u4e4b\u540e\uff0c\u5c31\u53ef\u4ee5\u5f97\u5230\u6211\u4eec\u60f3\u8981\u7684\u6570\u636e\u4e86~~<\/p>\n
\u53c2\u8003\u94fe\u63a5\uff1a<\/h6>\n\n- python traverse nested dictionary – AOL Search Results<\/a><\/li>\n
- python – Loop through all nested dictionary values? – Stack Overflow<\/a><\/li>\n
- dictionary – Iterating over Dictionaries…For Loops in Python – Stack Overflow<\/a><\/li>\n
- create a double dictionary in python \u00bb from the desk of stinkpot<\/a><\/li>\n
- Generic way to create nested dictionary from flat list in python – Stack Overflow<\/a><\/li>\n
- Python\u77e5\u8bc6\u79ef\u7d2f_0 | ASPIRE<\/a><\/li>\n
- \u4e00\u9053\u601d\u8003\u9898_0 | ASPIRE<\/a><\/li>\n
- Built-in Types \u2014 Python 2.7.9 documentation<\/a><\/li>\n
- Chapter 10. Nested data structures<\/a><\/li>\n
- data structures – What is the best way to implement nested dictionaries in Python? – Stack Overflow<\/a><\/li>\n
- dictionary – How to initialize nested dictionaries in Python – Stack Overflow<\/a><\/li>\n
- nested – Python dictionaries-How to keep the new value from overwriting the previous value? – Stack Overflow<\/a><\/li>\n
- Sorting dictionary keys in python – Stack Overflow<\/a><\/li>\n
- Access python nested dictionary items via a list of keys – Stack Overflow<\/a><\/li>\n
- Python \u4ee3\u7801\u6027\u80fd\u4f18\u5316\u6280\u5de7<\/a><\/li>\n
- Python\u9ad8\u7ea7\u7279\u6027\u4e4b\uff1aList Comprehensions\u3001Generator\u3001Dictionary and set comprehensions<\/a><\/li>\n
- nested – Python dictionaries-How to keep the new value from overwriting the previous value?\u00a0– Stack Overflow<\/a><\/li>\n
- Python Recursive remove nested dict – AOL Search Results<\/a><\/li>\n
- Some more python recursion examples – SaltyCrane Blog<\/a><\/li>\n
- Python: How RECURSIVELY remove None values from a NESTED data structure (lists and dictionaries)? – Stack Overflow<\/a><\/li>\n
- scripting – deleting items from a dictionary while iterating over it – Stack Overflow<\/a><\/li>\n
- python – Elegant way to remove fields from nested dictionaries – Stack Overflow<\/a><\/li>\n
- Python nested dict length – AOL Search Results<\/a><\/li>\n
- recursion – How to get keys from nested dictionary of arbitrary length in Python – Stack Overflow<\/a><\/li>\n
- python – How to count all elements in a nested dictionary? – Stack Overflow<\/a><\/li>\n
- Reading\/writing a list of nested dictionaries to\/from a CSV file (Python) – Stack Overflow<\/a><\/li>\n
- python – How to write a nested dictionary to json – Stack Overflow<\/a><\/li>\n
- output – In Python, how do I write the contents of nested dictionaries to a file in a certain format? – Stack Overflow<\/a><\/li>\n<\/ul>\n
=END=<\/p>\n","protected":false},"excerpt":{"rendered":"
=Start= \u7f18\u7531\uff1a \u4e4b\u524d\u5c31\u505a\u8fc7\u7f51\u7ad9\u8bbf\u95ee\u65e5\u5fd7\u7684\u5206\u6790\uff0c\u5206\u522b\u7528\u7684\u662fawk\u548cPython\uff0c\u4f46\u90fd\u6bd4\u8f83\u7b80\u5355\uff0c\u53ea\u662f\u7b80\u5355\u9009 […]<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[23,11,7,12],"tags":[74,494,8],"class_list":["post-2090","post","type-post","status-publish","format-standard","hentry","category-knowledgebase-2","category-linux","category-programing","category-tools","tag-awk","tag-nested","tag-python"],"views":4701,"_links":{"self":[{"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/posts\/2090","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/comments?post=2090"}],"version-history":[{"count":2,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/posts\/2090\/revisions"}],"predecessor-version":[{"id":3954,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/posts\/2090\/revisions\/3954"}],"wp:attachment":[{"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/media?parent=2090"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/categories?post=2090"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/tags?post=2090"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}