自作検索システムへのみちのり_001
package com.cocolog_nifty.kjunichi;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
public class GetETag {
public static int maxThread = 100;
public static int maxRecords = maxThread * 10;
public void execute() {
GetETagClient client;
client = new GetETagClient[maxThread];
Connection conn = null;
Statement stmt = null;
try {
Class.forName("org.postgresql.Driver");
} catch (Exception ex) {
System.out.println(ex);
}
try {
conn = DriverManager.getConnection(
"jdbc:postgresql://192.168.0.178/websearch", "junichi",
"");
} catch (Exception ex) {
System.out.println(ex);
}
boolean isLoop = true;
int count = 0;
while (isLoop) {
System.out.println("count = " + count);
count++;
if (count > 46000) {
isLoop = false;
}
// maxRecords分レコードを取得する
try {
stmt = conn.createStatement();
// SQL文の作成
String sql = "select urlid,url from url where status is null order by urlid limit "
+ maxRecords;
System.out.println("sql = " + sql);
// 検索実行
ResultSet rs = stmt.executeQuery(sql);
System.out.println("rs = " + rs);
if (rs != null) {
int i = 0;
while (rs.next()) {
MyUrl myUrl = new MyUrl();
// urlid
myUrl.setUrlid(rs.getObject(1).toString());
myUrl.setUrl(rs.getObject(2).toString());
vecUrl.add(myUrl);
System.out.println(rs.getObject(1));
System.out.println(":" + rs.getObject(2));
i++;
}
if (i < maxRecords) {
// 取得件数がmaxRecords以下の場合は、ループを終了する
isLoop = false;
System.out.println("i = " + i);
}
System.out.println("i = " + i);
}
rs.close();
} catch (Exception ex) {
ex.printStackTrace();
System.out.println(ex);
}
// ?取得したレコードをmaxThread分ずつ実行する
int index = 0;
int thCount = -1;
int clientCount = -1;
if(vecUrl.size() < maxRecords) {
thCount = 1;
clientCount = vecUrl.size();
} else {
thCount = maxRecords / maxThread;
clientCount = client.length;
}
for (int j = 0; j < thCount; j++) {
for (int i = 0; i < clientCount; i++) {
MyUrl myUrl = (MyUrl)vecUrl.get(index++);
client[i] = new GetETagClient();
client[i].setUrl(myUrl.getUrl());
client[i].setUrlid(myUrl.getUrlid());
}
// ヘッダ情報を取得する
for (int i = 0; i < clientCount; i++) {
client[i].start();
}
for (int i = 0; i < clientCount; i++) {
// 結果を集める
try {
client[i].join(1000 * 10);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println(count + "," + i + ":"
+ client[i].getUrlid() + " "
+ client[i].getUrl() + ":("
+ client[i].getStatusCode() + ") "
+ client[i].getETag());
PreparedStatement pstmt = null;
try {
// SQL文の作成
String sql = "update url set status=?, etag=? where url=?";
pstmt = conn.prepareStatement(sql);
// パラメータの設定
int col = 1;
// HTTPステータス
pstmt.setString(col++, new Integer(client[i]
.getStatusCode()).toString());
// ETag
pstmt.setString(col++, client[i].getETag());
// Url
pstmt.setString(col++, client[i].getUrl());
// 更新実行
int cnt = pstmt.executeUpdate();
if (cnt != 1) {
// 更新失敗
System.out.println("Oops ! cnt = " + cnt);
}
} catch (Exception ex) {
System.out.println(ex);
}
}
}
}
}
/**
* @param args
*/
public static void main(String args) {
GetETag etag = new GetETag();
etag.execute();
}
}
