{"id":5412,"date":"2023-03-25T08:55:00","date_gmt":"2023-03-25T00:55:00","guid":{"rendered":"https:\/\/ixyzero.com\/blog\/?p=5412"},"modified":"2023-03-22T19:59:37","modified_gmt":"2023-03-22T11:59:37","slug":"%e7%94%a8sql%e4%bb%8esql%e6%9f%a5%e8%af%a2%e8%af%ad%e5%8f%a5%e4%b8%ad%e6%8f%90%e5%8f%96%e8%a1%a8%e5%90%8d%e4%bf%a1%e6%81%af","status":"publish","type":"post","link":"https:\/\/ixyzero.com\/blog\/archives\/5412.html","title":{"rendered":"\u7528SQL\u4eceSQL\u67e5\u8be2\u8bed\u53e5\u4e2d\u63d0\u53d6\u8868\u540d\u4fe1\u606f"},"content":{"rendered":"\n<p>=Start=<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u7f18\u7531\uff1a<\/h4>\n\n\n\n<p>\u7b80\u5355\u6574\u7406\u4e00\u4e0b\u524d\u6bb5\u65f6\u95f4\u505a\u7684\u7528Hive SQL\u4eceSQL\u67e5\u8be2\u8bed\u53e5\u4e2d\u63d0\u53d6\u8868\u540d\u7684\u5185\u5bb9\uff0c\u65b9\u4fbf\u540e\u9762\u6709\u9700\u8981\u7684\u65f6\u5019\u53c2\u8003\uff08\u4e4b\u524d\u8bb0\u5f55\u8fc7\u7528Python\u548cJava\u4eceSQL\u4e2d\u63d0\u53d6\u8868\u540d\u3001\u5217\u540d\u7684\u529f\u80fd\uff0c\u4f46\u8fd9\u91cc\u51fa\u4e8e\u65b9\u4fbf\u8d77\u89c1\u2014\u2014\u4e0d\u60f3\u5199UDF\uff0c\u51c6\u5907\u76f4\u63a5\u5728ETL\u91cc\u7528Hive SQL\u6765\u5b9e\u73b0\uff09\u3002<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u6b63\u6587\uff1a<\/h4>\n\n\n\n<h5 class=\"wp-block-heading\">\u53c2\u8003\u89e3\u7b54\uff1a<\/h5>\n\n\n\n<h5 class=\"wp-block-heading\">\u7b80\u5355\u5feb\u901f\u7684\u4e00\u4e2a\u5b9e\u73b0<\/h5>\n\n\n\n<pre class=\"wp-block-code\"><code>-- \u4ecemysql\u67e5\u8be2\u8bed\u53e5\u4e2d\u63d0\u53d6\u8868\u540d\n\n,substring_index(split(lower(query_sql),'from\\\\s{1,}|where')&#91;1],' ',1) as tbl_name\n\n,split(split(lower(query_sql),'from\\\\s{1,}|where')&#91;1],'\\\\s{1,}')&#91;0] as tbl_name\n\n-- \u5df2\u77e5\u95ee\u9898\uff1a\n-- \u5bf9\u4e8e\u6709 join \u7b49\u591a\u8868\u8fde\u63a5\u7684\u65e0\u6cd5\u5c06\u6240\u6709\u8868\u540d\u90fd\u62ff\u5230\uff1b\n-- \u5bf9\u4e8e\u6574\u4f53\u8bed\u53e5\u4e2d\u5b58\u5728\u6ce8\u91ca\u8bed\u53e5\u7684\u65e0\u6cd5\u5f88\u597d\u8bc6\u522b\uff08\u9664\u975e\u662f\u6240\u6709\u6ce8\u91ca\u8bed\u53e5\u90fd\u653e\u5230\u4e86\u6700\u540e\uff09\uff1b<\/code><\/pre>\n\n\n\n<h5 class=\"wp-block-heading\">\u5b8c\u6574\u7684\u63d0\u53d6\u8868\u540d\u7684\u601d\u8def<\/h5>\n\n\n\n<pre class=\"wp-block-code\"><code>\/*\n\u4ece\u5b8c\u6574\u7684SQL\u8bed\u53e5\u5b57\u7b26\u4e32\u4e2d\u63d0\u53d6\u8868\u540d\u7684\u601d\u8def\u548c\u6b65\u9aa4\n\n1. \u5148\u7528\u6b63\u5219\u8868\u8fbe\u5f0f\u5c06SQL\u67e5\u8be2\u4e2d\u7684\u6ce8\u91ca\u884c\u66ff\u6362\u4e3a\u7a7a\uff0c\u751f\u6210\u771f\u6b63\u6709\u6548\u7684SQL\u8bed\u53e5\uff0c\u4e3a\u540e\u9762\u7684\u8868\u540d\u63d0\u53d6\u6392\u9664\u5e72\u6270\n2. \u5bf9\u6ca1\u6709\u591a\u8868\u8fde\u63a5\u7684\u67e5\u8be2\u8fdb\u884c\u63d0\u53d6\uff08\u53ef\u4ee5\u4fdd\u8bc1\u63d0\u53d6\u7684\u51c6\u786e\u7387\uff09\uff1b\n3. \u518d\u5bf9\u5f53\u524d\u5df2\u77e5\u7684\u51e0\u79cd\u8868\u8fde\u63a5\u65b9\u5f0f\u5c1d\u8bd5\u63d0\u53d6\uff0c\u53ef\u4ee5\u5c06\u63d0\u53d6\u51fa\u6765\u7684\u7ed3\u679c\u653e\u5728\u591a\u884c\u4e2d\uff0c\u65b9\u4fbf\u540e\u9762\u7684\u5173\u8054\u5339\u914d\uff1b\n4. \u6709\u6570\u636e\u5e93\u540d\u7684\u4e5f\u7528\u4e00\u4e2a\u5b57\u6bb5\u63d0\u53d6\u51fa\u6765\u5907\u7740\uff1b\n5. \u589e\u52a0\u51e0\u4e2a\u5b57\u6bb5\u7528\u6765\u7ed9SQL\u6253\u6807\u7b7e\uff08\u6240\u6709\u5217\/where\u6761\u4ef6\/limit\u6570\u91cf\uff09\uff1b\n*\/\n\n\/*\n\u6839\u636e\u62bd\u6837\u7684\u60c5\u51b5\u6765\u770b\uff0cSQL\u8bed\u53e5\u4e2d\uff1a\nfrom \u4e4b\u540e\u7684\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u662f table1\njoin \u4e4b\u540e\u7684\u7b2c\u4e00\u4e2a\u5b57\u6bb5\u662f table2\/3\/...\n\u6b63\u5e38\u60c5\u51b5\u4e0b\u53ea\u7528\u8003\u8651 2 \u4e2a\u8868\u5173\u8054\u7684\u60c5\u51b5\uff0c\u56e0\u4e3a 3 \u4e2a\u6216\u4ee5\u4e0a\u8868\u5173\u8054\u7684\u6bd5\u7adf\u662f\u5c11\u6570\uff0c\u4e0d\u8fc7\u53ef\u4ee5\u7edf\u8ba1\u4e00\u4e0b\u6bd4\u7387\uff0c\u65b9\u4fbf\u6700\u540e\u505a\u51b3\u7b56\n*\/\n\n\n-- todo: add some tag of sql, such as has *\/where\/limit\/--\n-- \u6709\u51e0\u79cd\u65b9\u6848\u5728\u4e00\u5217\u91cc\u9762\u8868\u793a\u591a\u4e2a\u4fe1\u606f\uff0c\u6bd4\u5982\uff1a\n-- \u6570\u5b57 1111111 \u6bcf\u4e00\u4f4d\u75281\/0\u6765\u8868\u793a\u8fd9\u4e2a\u4f4d\u7f6e\u4e0a\u7684\u76ee\u6807\u5b57\u7b26\u4e32\u662f\u5426\u5b58\u5728\uff0c\u5224\u65ad\u8d77\u6765\u662f\u7528\u53d6\u6a21\u7b49\u65b9\u5f0f\uff0c\u4f46\u8fd8\u6ca1\u6709\u5b8c\u5168\u60f3\u597d\n-- \u5b57\u7b26\u4e32 abcdef \u6bcf\u4e00\u4e2a\u4e0d\u540c\u7684\u5b57\u7b26\u8868\u793a\u5e0c\u671b\u67e5\u627e\u7684\u76ee\u6807\u5b57\u7b26\u4e32\u662f\u5426\u5b58\u5728\uff0c\u5224\u65ad\u8d77\u6765\u4e5f\u5f88\u65b9\u4fbf\uff0c\u76f4\u63a5 locate\/findstr\/like \u5c31\u884c<\/code><\/pre>\n\n\n\n<h5 class=\"wp-block-heading\">\u5206\u6790\u8fc7\u7a0b\u4e2d\u7684\u4e00\u4e9b\u4ee3\u7801\u7247\u6bb5<\/h5>\n\n\n\n<pre class=\"wp-block-code\"><code>-- select.+?\\n\n-- \u5c1d\u8bd5\u7528\u6362\u884c\u7b26\u8fdb\u884c\u5207\u5206\uff0c\u7136\u540e\u5bf9\u6bcf\u4e00\u4e2a\u7ed3\u679c\u8fdb\u884c\u5224\u65ad\uff0c\u4e0d\u8fc7\u8fd9\u4e2a\u53ea\u5bf9\u90e8\u5206SQL\u624d\u80fd\u751f\u6548\uff0c\u548c\u5177\u4f53\u7684SQL\u7f16\u5199\u4e60\u60ef\u6709\u5173(\u751f\u4ea7\u73af\u5883\u4e2d\u5404\u79cd\u53ef\u80fd\u6027\u90fd\u6709\u53ef\u80fd\u51fa\u73b0\uff0c\u53ea\u8981\u5b83\u7b26\u5408\u57fa\u672c\u7684\u89c4\u8303\u539f\u7406\u5c31\u884c\uff0c\u6240\u4ee5\u6839\u672c\u4e0a\u8fd8\u662f\u4ece\u8bed\u6cd5\u89e3\u6790\u4e0a\u7740\u624b\u662f\u6700\u9760\u8c31\u7684)\nlateral view explode(split(query_sql,'\\\\n{1,}')) pi as sql1\n\n-- \u5982\u4f55\u5224\u65ad\u4e00\u4e2aSQL\u8bed\u53e5\u4e2d\u662f\u5426\u5305\u542b\u8868\u8fde\u63a5\u7684\u64cd\u4f5c\n-- \u5f53\u524d\u60f3\u5230\u76843\u4e2a\u5224\u65ad\u6761\u4ef6\u662f join\/on\/from\u4e4b\u540ewhere\u4e4b\u524d\u6709\u6ca1\u6709\u9017\u53f7\nand (query_sql rlike '\\\\s{1,}join\\\\s{1,}' or query_sql rlike '\\\\s{1,}on\\\\s{1,}' or (query_sql like '%where%' and split(lower(query_sql),'from\\\\s{1,}|where\\\\s{1,}')&#91;1] like '%,%'))\n\n\n-- \u5207\u6362\u5904\u7406\u601d\u8def\uff0c\u5148\u5c06SQL\u8bed\u53e5\u505a\u683c\u5f0f\u5316\u5904\u7406\uff0c\u7136\u540e\u518d\u901a\u8fc7split\u7b49\u65b9\u5f0f\u8fdb\u884c\u5207\u5206\u63d0\u53d6\uff0c\u6bd4\u5982\u5148\u5c06\u8fde\u7eed\u7684\u591a\u4e2a\u7a7a\u767d\u7b26\u66ff\u6362\u6210\u7a7a\u683c\nregexp_replace(query_sql,'\\\\s{1,}',' ')\n\n\n-- \u5148\u7528\u6b63\u5219\u8868\u8fbe\u5f0f\u5c06SQL\u67e5\u8be2\u4e2d\u7684\u6ce8\u91ca\u884c\u66ff\u6362\u4e3a\u7a7a\uff0c\u751f\u6210\u771f\u6b63\u6709\u6548\u7684SQL\u8bed\u53e5\uff0c\u6548\u679c\u5f88\u597d\n,regexp_replace(query_sql,'--.+?\\\\n{1,}','') as real_sql<\/code><\/pre>\n\n\n\n<h5 class=\"wp-block-heading\">\u4e00\u4e2a\u5b8c\u6574\u7684\u4eceSQL\u63d0\u53d6\u8868\u540d\u7684\u6837\u4f8b<\/h5>\n\n\n\n<pre class=\"wp-block-code\"><code>-- \u4e0b\u9762\u7684SQL\u901a\u8fc7\u591a\u5c42\u5b50\u67e5\u8be2\uff0c\u524d\u671f\u901a\u8fc7\u5254\u9664\u6ce8\u91ca\u8bed\u53e5\u548c\u7edf\u4e00\u7a7a\u767d\u7b26\u8fdb\u884c\u89c4\u8303\u5316\u5904\u7406\uff0c\u7136\u540e\u901a\u8fc7\u5bf9 from|join \u7b49\u548c\u8868\u5728\u4e00\u8d77\u7684\u5173\u952e\u5b57\u8fdb\u884c split \u5207\u5206\n-- \u4ece\u800c\u5f97\u5230\u591a\u4e2a\u4ee5\u8868\u5f00\u5934\u7684\u5b57\u7b26\u4e32\uff0c\u501f\u52a9 explode \u5c06\u4e00\u884c\u8f6c\u6210\u591a\u884c\uff0c\u7136\u540e\u518d\u63d0\u53d6\u8868\u540d\uff0c\u5e76\u7528 where \u6761\u4ef6\u5bf9\u65e0\u6548\u5207\u5206\u6570\u636e\u8fdb\u884c\u6392\u9664\n-- \u6700\u7ec8\u5f97\u5230\u7684\u6570\u636e\u683c\u5f0f\u4e3aSQL\u4e2d\u4f7f\u7528\u5230\u7684\u6bcf\u4e00\u4e2a\u8868\u4e3a\u4e00\u884c\u6570\u636e\uff0c\u65b9\u4fbf\u540e\u9762\u505a\u5173\u8054\u5206\u6790\nSELECT\n`user_name`\n,`time_str`\n,join_tag\n,join_tag2\n,join_cnt\n,tbl_x\n,tbl_namex\n,real_sql\n,`query_sql`\n,`request_body`\nFROM\n(SELECT\n`user_name`\n,`time_str`\n,case\nwhen real_sql like '% join %' then 1\nwhen real_sql like '% on %' then 2\nelse 0\nend as join_tag\n,if(split(real_sql,'from ')&#91;1] like '%,%',1,0) as join_tag2\n,(length(real_sql) - length(replace(real_sql,' join ',''))) \/ length(' join ') as join_cnt\n,tbl_x\n,substring_index(tbl_x,' ',1) as tbl_namex\n,real_sql\n,`query_sql`\n,`request_body`\nFROM\n(SELECT\n-- `dt`,\n`user_name`\n,`time_str`\n,lower(regexp_replace(regexp_replace(query_sql,'--.+?\\\\n{1,}',''),'\\\\s{1,}',' ')) as real_sql\n,`query_sql`\n,`request_body`\n-- todo: add some tag of sql, such as has *\/where\/limit\/--\n-- ,if(query_sql like '%--%',1,0) as tag_comment\nFROM\nlog.mysql_search_op\nWHERE\ndt between '20230315' and '20230315'\nand query_sql like '%--%'\n)x1\nlateral view explode(split(real_sql,' from | join ')) pi as tbl_x\n)x2\nwhere join_tag &gt; 0 and tbl_namex not in ('','select')\nLIMIT 1000\n;<\/code><\/pre>\n\n\n\n<h5 class=\"wp-block-heading\">\u53c2\u8003\u94fe\u63a5\uff1a<\/h5>\n\n\n\n<p>\u5982\u4f55\u7528Python\u4eceSQL\u4e2d\u63d0\u53d6\u51fa\u6d89\u53ca\u5230\u7684\u8868\u540d\u3001\u5217\u540d\uff1f<br><a href=\"https:\/\/ixyzero.com\/blog\/archives\/4404.html\">https:\/\/ixyzero.com\/blog\/archives\/4404.html<\/a><\/p>\n\n\n\n<p>How to extract tables names in a SQL script?<br><a href=\"https:\/\/stackoverflow.com\/questions\/49773059\/how-to-extract-tables-names-in-a-sql-script\">https:\/\/stackoverflow.com\/questions\/49773059\/how-to-extract-tables-names-in-a-sql-script<\/a><\/p>\n\n\n\n<p>\u5982\u4f55\u7528Java\u4eceSQL\u4e2d\u63d0\u53d6\u51fa\u6d89\u53ca\u5230\u7684\u8868\u540d\u3001\u5217\u540d\uff1f<br><a href=\"https:\/\/ixyzero.com\/blog\/archives\/4405.html\">https:\/\/ixyzero.com\/blog\/archives\/4405.html<\/a><\/p>\n\n\n\n<p>MySQL\u591a\u8868join\u8fde\u63a5\u67e5\u8be2<br><a href=\"https:\/\/ixyzero.com\/blog\/archives\/4211.html\">https:\/\/ixyzero.com\/blog\/archives\/4211.html<\/a><\/p>\n\n\n\n<p>Hive SQL\u5b66\u4e60\u6574\u7406<br><a href=\"https:\/\/ixyzero.com\/blog\/archives\/4245.html\">https:\/\/ixyzero.com\/blog\/archives\/4245.html<\/a><\/p>\n\n\n\n<p>SQL \u67e5\u8be2\u5e76\u4e0d\u662f\u4ece SELECT \u5f00\u59cb\u7684(SQL queries don\u2019t start with SELECT)<br><a href=\"https:\/\/mp.weixin.qq.com\/s\/5BF12rIs1QXQpX4vpKkL0Q\">https:\/\/mp.weixin.qq.com\/s\/5BF12rIs1QXQpX4vpKkL0Q<\/a><\/p>\n\n\n\n<p>=END=<\/p>\n","protected":false},"excerpt":{"rendered":"<p>=Start= \u7f18\u7531\uff1a \u7b80\u5355\u6574\u7406\u4e00\u4e0b\u524d\u6bb5\u65f6\u95f4\u505a\u7684\u7528Hive SQL\u4eceSQL\u67e5\u8be2\u8bed\u53e5\u4e2d\u63d0\u53d6\u8868\u540d\u7684\u5185\u5bb9\uff0c\u65b9\u4fbf\u540e\u9762\u6709 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[14,23,7],"tags":[1652,1261,421,1262,1560,1920],"class_list":["post-5412","post","type-post","status-publish","format-standard","hentry","category-database","category-knowledgebase-2","category-programing","tag-etl","tag-hive","tag-join","tag-sql","tag-udf","tag-1920"],"views":2353,"_links":{"self":[{"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/posts\/5412","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/comments?post=5412"}],"version-history":[{"count":1,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/posts\/5412\/revisions"}],"predecessor-version":[{"id":5413,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/posts\/5412\/revisions\/5413"}],"wp:attachment":[{"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/media?parent=5412"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/categories?post=5412"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/ixyzero.com\/blog\/wp-json\/wp\/v2\/tags?post=5412"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}