Showing posts with label Tool. Show all posts
Showing posts with label Tool. Show all posts

Sunday, July 24, 2011

DML SQL解析器(支持标准的数据库SQL)

在前一家公司的时候,需要审核一些SQL语句,因此就想弄一个自动审核的工具,当时想写一个语法解析器,但一直没有完成,最近在开源代码和javacc的基础上整合了一个DML的SQL解析器,目前支持的语法关键字有:


ALL,AND,ANY,AS,ASC,AVG,BETWEEN,BY,COUNT,
DELETE,DESC,DISTINCT,EXISTS,FROM,GROUP,
HAVING,IN,INSERT,INTERSECT,INTO,IS,LIKE,
MAX,MIN,MINUS,NOT,NULL,OF,ONLY,OR,ORDER,
,SELECT,SET,SUM,UNION,UPDATE,VALUES
,WHERE
SQL解析结果包括语法解析和验证、表名获取、where语句的获取。

以下是测试代码:

public static void main(String args[]) throws ParseException {
XsqlParser p = null;
try {
String filename = "D://zhongli//parserbooks//xsql_test//test//TEST.in";// args[0]
p = new XsqlParser(new DataInputStream(
new FileInputStream(filename)));
} catch (FileNotFoundException e) {
System.out.println("/* File " + args[0]
+ " not found. Reading from stdin */");
p = new XsqlParser(System.in);
}
XStatement st = null;
while ((st = p.readStatement()) != null) {
System.out.println(st.toString() + ";");
}

System.out.println("exit;");
System.out.println("/* Parse Successful */");

// for string SQL Statement
st = null;
p
.setStatement("SELECT EMPLOYEEIDNO as NO FROM EMPLOYEESTATISTICSTABLE"
+ " WHERE POSITION = 'Manager' AND (SALARY > 50000 OR BENEFIT > 10000);");
if ((st = p.readStatement()) != null) {
System.out.println(st.toString() + ";");
System.out.println(st.getTable());
System.out.println(st.getWhere());
}
// for join
st = null;
p.setStatement("SELECT DISTINCT SELLERID, OWNERLASTNAME, OWNERFIRSTNAME"
+ " FROM ANTIQUES, ANTIQUEOWNERS"
+ " WHERE SELLERID = OWNERID"
+ " ORDER BY OWNERLASTNAME, OWNERFIRSTNAME, OWNERID;");
if ((st = p.readStatement()) != null) {
System.out.println(st.toString() + ";");
System.out.println(st.getTable());
System.out.println(st.getWhere());
}
} // main ends here
通过测试验证支持的SQL如下:

select * from TableName;
select FirstName, LastName, Address, City, State from EmployeeAddressTable;
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where (SALARY >= 50000);
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where (POSITION = 'Manager');
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where ((SALARY > 40000) AND (POSITION = 'Staff'));
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where ((SALARY < 40000) OR (BENEFITS < 10000));
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where (((POSITION = 'Manager') AND (SALARY > 60000)) OR (BENEFITS > 12000));
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where ((POSITION = 'Manager') AND ((SALARY > 50000) OR (BENEFIT > 10000)));
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where (POSITION IN ('Manager', 'Staff'));
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where (SALARY BETWEEN 30000 AND 50000);
select EMPLOYEEIDNO from EMPLOYEESTATISTICSTABLE where (SALARY NOT BETWEEN 30000 AND 50000);
select EMPLOYEEIDNO from EMPLOYEEADDRESSTABLE where (LASTNAME LIKE 'L%');
select OWNERLASTNAME, OWNERFIRSTNAME from ANTIQUEOWNERS, ANTIQUES where ((BUYERID = OWNERID) AND (ITEM = 'Chair'));
select ANTIQUEOWNERS.OWNERLASTNAME, ANTIQUEOWNERS.OWNERFIRSTNAME from ANTIQUEOWNERS, ANTIQUES where ((ANTIQUES.BUYERID = ANTIQUEOWNERS.OWNERID) AND (ANTIQUES.ITEM = 'Chair'));
select distinct SELLERID, OWNERLASTNAME, OWNERFIRSTNAME from ANTIQUES, ANTIQUEOWNERS where (SELLERID = OWNERID) order by OWNERLASTNAME ASC, OWNERFIRSTNAME ASC, OWNERID ASC;
select OWN.OWNERLASTNAME Last Name, ORD.ITEMDESIRED Item Ordered from ORDERS ORD, ANTIQUEOWNERS OWN where ((ORD.OWNERID = OWN.OWNERID) AND (ORD.ITEMDESIRED IN (select ITEM from ANTIQUES)));
select SUM(SALARY), AVG(SALARY) from EMPLOYEESTATISTICSTABLE;
select MIN(BENEFITS) from EMPLOYEESTATISTICSTABLE where (POSITION = 'Manager');
select COUNT(*) from EMPLOYEESTATISTICSTABLE where (POSITION = 'Staff');
select SELLERID from ANTIQUES, ANTVIEW where (ITEMDESIRED = ITEM);
insert into ANTIQUES values (21, 01, 'Ottoman', 200.00);
insert into ANTIQUES(BUYERID,SELLERID,ITEM) values (01, 21, 'Ottoman');
delete from ANTIQUES where (ITEM = 'Ottoman');
delete from ANTIQUES where ((ITEM = 'Ottoman') AND (BUYERID = 01) AND (SELLERID = 21));
update ANTIQUES set PRICE=500.00 where (ITEM = 'Chair');
select BUYERID, MAX(PRICE) from ANTIQUES group by BUYERID;
select BUYERID, MAX(PRICE) from ANTIQUES group by BUYERID having (PRICE > 1000);
select OWNERID from ANTIQUES where (PRICE > (select (AVG(PRICE) + 100) from ANTIQUES));
select OWNERLASTNAME from ANTIQUEOWNERS where (OWNERID = (select distinct BUYERID from ANTIQUES));
update ANTIQUEOWNERS set OWNERFIRSTNAME='John' where (OWNERID = (select BUYERID from ANTIQUES where (ITEM = 'Bookcase')));
select OWNERFIRSTNAME, OWNERLASTNAME from ANTIQUEOWNERS where (EXISTS (select * from ANTIQUES where (ITEM = 'Chair')));
select BUYERID, ITEM from ANTIQUES where (PRICE >= ALL (select PRICE from ANTIQUES));
select BUYERID from ANTIQUEOWNERS UNION (select OWNERID from ORDERS);
select OWNERID, 'is in both Orders & Antiques' from ORDERS, ANTIQUES where (OWNERID = BUYERID) UNION (select BUYERID, 'is in Antiques only' from ANTIQUES where (BUYERID NOT IN (select OWNERID from ORDERS)));
select * from GRADES where (GRADE = (((2 + 3) - 4) + 5));
select * from GRADES where (GRADE = ((2 + (3 * 4)) - 5));
select * from table1 where (col1 IS NULL);
select * from table1 where (avg(col1) IS NOT NULL);

select EMPLOYEEIDNO NO from EMPLOYEESTATISTICSTABLE where ((POSITION = 'Manager') AND ((SALARY > 50000) OR (BENEFIT > 10000)));
EMPLOYEESTATISTICSTABLE
((POSITION = 'Manager') AND ((SALARY > 50000) OR (BENEFIT > 10000)))

select distinct SELLERID, OWNERLASTNAME, OWNERFIRSTNAME from ANTIQUES, ANTIQUEOWNERS where (SELLERID = OWNERID) order by OWNERLASTNAME ASC, OWNERFIRSTNAME ASC, OWNERID ASC;

ANTIQUES,ANTIQUEOWNERS
(SELLERID = OWNERID)

获取源码和可执行二进制jar包URL如下:
源码:
http://www.box.net/shared/2urubkpngx9vbolmp5pi

二进制jar包:

http://www.box.net/shared/0gphjv9p71x18ntrfd8o

目前一些数据库的特殊语法没有实现,如果需要实现的,可以email联系我。




Friday, July 1, 2011

Data sync tool for database(oracle2oracle, oracle2mysql,mysql2oracle...)

   最近项目需要快速的拖取oracle大容量表的数据到MYSQL中,因此写了一个多线程并发的java数据拖取工具,在此基础上抽象了一个比较通用的工具,目前是基于JDBC方式,因此支持oracle2oracle,oracle2mysql,mysql2mysql,mysql2oracle,mysql2sqlserver,sqlserver 2mysql等等,通过测试目前有oracle2mysql,mysql2oracle,支持windows和Linux。下面说明下使用方法和测试的用例。
     第一步,获取可以运行的二进制代码,可以从下面的URL获取二进制执行包: 
    第二步,配置需要同步的数据库和表,打开conf文件下的config.properties,conf文件下包含了其他数据同步的模版,下面举例多线程同步mysql的表到oracle中,config.properties是如何配置的。
#for source database parameters
source.dataSource.initialSize=10
source.dataSource.maxIdle=20
source.dataSource.minIdle=5
source.dataSource.maxActive=100
source.dataSource.maxWait=120000
source.jdbc.driverClassName=com.mysql.jdbc.Driver
source.jdbc.url=jdbc:mysql://10.224.56.188:3306/meetingdb?autoReconnect=true&characterEncoding=UTF-8
source.jdbc.username=test
source.jdbc.password=pass
#Target sync data threadNum=source.database.threadNum
source.database.threadNum=10
source.database.selectSql=select * from  test  where mod(CAST(FNV_64(PATHALIASID) AS UNSIGNED),#threadNum#)=?
#you can input many commands and split by ";" ,你可以打开注释加一些session级别的优化命令
#source.database.sessionCommand=ALTER SESSION SET DB_FILE_MULTIBLOCK_READ_COUNT=128;
#for target jdbc parameters
target.dataSource.initialSize=10
target.dataSource.maxIdle=20
target.dataSource.minIdle=5
target.dataSource.maxActive=100
target.dataSource.maxWait=120000
target.jdbc.driverClassName=oracle.jdbc.driver.OracleDriver
target.jdbc.url=jdbc:oracle:thin:@10.224.56.189:1521:PERFCON1
target.jdbc.username=test
target.jdbc.password=pass
target.database.insertSql=insert into test2(PATHALIASID,PATH,CREATETIME,LASTMODIFIEDTIME,OBJECTPREFIX,PATHMD5ID,COLLIDESWITH) values(?,?,?,?,?,?,?)
target.database.commitNum=1000
第三步,运行dataSync.sh或者dataSync.bat,开始同步数据。

具体的测试的用例如下:

Case1:oracle to mysql with multi_threads
source table:test
target table:test2
source.database.selectSql=select * from  test  where ORA_HASH(PATHALIASID,#threadNum#)=?

exeSql=select * from  test where ORA_HASH(PATHALIASID,10)=10
...
exeSql=select * from  test  where ORA_HASH(PATHALIASID,10)=0

mysql> select count(*) from test2;
+----------+
| count(*) |
+----------+
|    218850 |
+----------+

Case 2:mysql to oracle with multi_threads
source table:test
target table:test2
source.database.selectSql=select * from  test where mod(CAST(FNV_64(PATHALIASID) AS UNSIGNED),#threadNum#)=?




exeSql=select * from  test where mod(CAST(FNV_64(PATHALIASID) AS UNSIGNED),10)=10
...
exeSql=select * from  test where mod(CAST(FNV_64(PATHALIASID) AS UNSIGNED),10)=0
SQL>select count(*) from test2;
  COUNT(*)
----------
     218850
 
case3:mysql to oracle with single process.
source.database.threadNum=1
source.database.selectSql=select * from  test
exeSql=select * from  test
SQL>select count(*) from test2;
  COUNT(*)
----------
     218850
 

Wednesday, March 2, 2011

Ibatis SQL Code performance check tool for oracle and mysql

when write the SQL,we want to check the SQL performance,
for example:
1)the SQL can't include select *.
2)the SQL must has certain column.
3)the SQL explain plan must not use full table scan or index scan.

we can define the rule as  following:

<?xml version="1.0" encoding="UTF-8"?> 
<rule-execution-set> 
  <name>daoCheckRules</name> 
  <description>DAO check Rule Execution Set</description> 
  <synonymn name="daoRuleParser" class="com.database.util.daoRuleParser" /> 
 
  <rule name="selectStar" description="Check SQL if has select * " > 
  <if leftTerm="daoRuleParser.getSql"  op="contains" rightTerm="*" /> 
  <then method="daoRuleParser.setCheckSelectStar" arg1="true" /> 
  </rule>
  
  <rule name="siteid" description="Check SQL if has no siteid column" > 
  <if leftTerm="daoRuleParser.getWhereClauseSql" op="notcontains" rightTerm="siteid" />
  <then method="daoRuleParser.setCheckSiteid" arg1="true"/> 
  </rule> 
 
  <rule name="checkOracleFullScan" description="Check SQL if oracle index or table full scan" > 
  <if leftTerm="daoRuleParser.getSqlExplan" op="contains" rightTerm="full scan" />
  <then method="daoRuleParser.setCheckOracleFullScan" arg1="true"/> 
  </rule>
 
  <rule name="checkMysqlFullScan" description="Check SQL if mysql index or table full scan" > 
  <if leftTerm="daoRuleParser.getSqlExplan" op="containsatleastone" rightTerm="[index,all]" />
  <then method="daoRuleParser.setCheckMysqlFullScan" arg1="true"/> 
  </rule>
 
  <rule name="checkReglevel" description="Check SQL if has wbxuser and coruser then must add REGLEVEL column" > 
  <if leftTerm="daoRuleParser.getAllTables" op="containsatleastone" rightTerm="[wbxuser,coruser]" />
  <if leftTerm="daoRuleParser.getWhereClauseSql" op="notcontains" rightTerm="reglevel"/>
  <then method="daoRuleParser.setCheckReglevel"  arg1="true"/> 
  </rule> 
</rule-execution-set>


then you can use the tool in your unit test case as following:

private static daoChecker daoCheck=null;
private static void initDaoChecker() throws DaoException
  {
              if(null==daoCheck){
              DaoManager.startTransaction();
              SqlMapDaoTransaction transaction = (SqlMapDaoTransaction)DaoManager.getLocalTransaction();
              SqlMap sqlMap=transaction.getSqlMap();
              daoCheck=new daoChecker(sqlMap);
              wbxDaoManager.commitTransaction();
              }      
  }
...
TestSuite suite = new TestSuite();
initDaoChecker();
suite.addTest(new TestUserPrivilegeDaoImpl("testFindUserRequest"));
daoCheck.daoCheckStart("findUserRequest",sc);

the unit test result as following:
 ERROR:         com.database.daoChecker     - Has table/index full scan in sql (findUserRequest)
 ERROR:         com.database.daoChecker     - Has no Reglevel field in  sql (findUserRequest)


now the tool is free, if you want to use the tool , you can contact with me, the blog can't input the jar package.