锘??xml version="1.0" encoding="utf-8" standalone="yes"?>
code: http://code.google.com/p/nutla/
涓銆佸畨瑁?br />
1銆?nbsp;瀹夎铏氭嫙鏈?Oracle VM VirtualBox4.0.4
2銆?nbsp;鍦ㄨ櫄鎷熸満涓嬪畨瑁?Red Hat 6.0
3銆?nbsp;瀹夎jdk jdk-6u24-linux-i586.bin 瀹夎璺緞涓猴細/home/nianzai/jdk1.6.0_24
4銆?nbsp;瀹夎hadoop hadoop-0.20.2.tar.gz 瀹夎璺緞涓猴細/home/nianzai/hadoop-0.20.2
5銆?nbsp;瀹夎zookeeper zookeeper-3.3.3.tar.gz 瀹夎璺緞涓猴細/home/nianzai/zookeeper-3.3.3
6銆?nbsp;瀹夎hbase hbase-0.90.2.tar.gz 瀹夎璺緞涓猴細/home/nianzai/hbase-0.90.2
浜屻侀厤緗?br />
1銆丩inux閰嶇疆
ssh-keygen –t rsa -P ''
cd .ssh
cp id_rsa.pub authorized_keys
/etc/hosts閲屽鍔?192.168.195.128 master
/etc/profile 閲屽鍔?br />
export JAVA_HOME=/home/nianzai/jdk1.6.0_24
export PATH=$PATH:$JAVA_HOME/bin
2銆乭adoop閰嶇疆
hadoop-env.sh
JAVA_HOME=/home/nianzai/jdk1.6.0._24
core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/home/nianzai/hadoop</value>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://master:9000</value>
</property>
</configuration>
hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>
mapred-site.xml
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>master:9001</value>
</property>
</configuration>
masters
master
sh hadoop namenode -format
sh start-all.sh
sh hadoop fs -mkdir input
3銆亃ookeeper閰嶇疆
zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/home/nianzai/zkdata
clientPort=2181
sh zkServer.sh start
4銆乭base閰嶇疆
hbase-env.sh
export JAVA_HOME=/home/nianzai/jdk1.6.0_24
export HBASE_MANAGES_ZK=false
灝唄base0.90.2 lib鐩綍涓媓adoop-core-0.20-append-r1056497.jar鍒犻櫎錛屾浛鎹㈡垚hadoop0.20.2 涓嬬殑hadoop-0.20.2-core.jar
hbase-site.xml
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://master:9000/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.master.port</name>
<value>60000</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>master</value>
</property>
</configuration>
regionservers
master
sh start-hbase.sh
Nut鏄竴涓狶ucene+Hadoop鍒嗗竷寮忓茍琛岃綆楁悳绱㈡鏋訛紝鑳藉鍗僄浠ヤ笂绱㈠紩鎻愪緵7*24灝忔椂鎼滅儲鏈嶅姟銆傚湪鏈嶅姟鍣ㄨ祫婧愯凍澶熺殑鎯呭喌涓嬭兘杈懼埌姣忕澶勭悊100涓囨鐨勬悳绱㈣姹傘?br />
Nut寮鍙戠幆澧冿細jdk1.6.0.23+lucene3.0.3+eclipse3.6.1+hadoop0.20.2+zookeeper3.3.2+hbase0.20.6+memcached+mongodb+linux
2銆佺壒鏂?br />
a銆佺儹鎻掓嫈
b銆佸彲鎵╁睍
c銆侀珮璐熻澆
d銆佹槗浣跨敤,涓庣幇鏈夐」鐩棤緙濋泦鎴?br />
e銆佹敮鎸佹帓搴?br />
f銆?*24鏈嶅姟
g銆佸け璐ヨ漿縐?
3銆佹悳绱㈡祦紼?br />
Nut鐢盜ndex銆丼earch銆丆lient銆丆ache鍜孌B浜旈儴鍒嗘瀯鎴愩?Cache瀹炵幇浜嗗memcached鐨勬敮鎸?DB瀹炵幇浜嗗hbase,mongodb鐨勬敮鎸?
Client澶勭悊鐢ㄦ埛璇鋒眰鍜屽鎼滅儲緇撴灉鎺掑簭銆係earch瀵硅姹傝繘琛屾悳绱紝Search涓婂彧鏀劇儲寮曪紝鏁版嵁瀛樺偍鍦―B涓紝Nut灝嗙儲寮曞拰瀛樺偍鍒嗙銆侰ache緙撳瓨鐨勬槸鎼滅儲鏉′歡鍜岀粨鏋滄枃妗d銆侱B瀛樺偍鐫鏁版嵁錛孋lient鏍規嵁鎼滅儲鎺掑簭緇撴灉,鍙栧嚭褰撳墠欏典腑鐨勬枃妗d浠嶥B涓婅鍙栨暟鎹?/p>
鐢ㄦ埛鍙戣搗鎼滅儲璇鋒眰緇欑敱Nut Client鏋勬垚鐨勯泦緹わ紝鐢辨煇涓狽ut Client鏍規嵁鎼滅儲鏉′歡鏌ヨCache鏈嶅姟鍣ㄦ槸鍚︽湁璇ョ紦瀛橈紝濡傛灉鏈夌紦瀛樻牴鎹紦瀛樼殑鏂囨。id鐩存帴浠嶥B璇誨彇鏁版嵁錛屽鏋滄病鏈夌紦瀛樺皢闅忔満閫夋嫨涓緇勬悳绱㈡湇鍔″櫒緇?Search Group i),灝嗘煡璇㈡潯浠跺悓鏃跺彂緇欒緇勬悳绱㈡湇鍔″櫒緇勯噷鐨刵鍙版悳绱㈡湇鍔″櫒錛屾悳绱㈡湇鍔″櫒灝嗘悳绱㈢粨鏋滆繑鍥炵粰Nut Client鐢卞叾鎺掑簭錛屽彇鍑哄綋鍓嶉〉鏂囨。id錛屽皢鎼滅儲鏉′歡鍜屽綋鍓嶆枃妗d緙撳瓨錛屽悓鏃朵粠DB璇誨彇鏁版嵁銆?/p>
4銆佺儲寮曟祦紼?br /> Hadoop Mapper/Reducer 寤虹珛绱㈠紩銆傚啀灝嗙儲寮曚粠HDFS鍒嗗彂鍒板悇涓儲寮曟湇鍔″櫒銆?br /> 瀵圭儲寮曠殑鏇存柊鍒嗕負涓ょ錛氬垹闄ゅ拰娣誨姞錛堟洿鏂板垎瑙d負鍒犻櫎鍜屾坊鍔狅級銆?br /> a銆佸垹闄?br /> 鍦℉DFS涓婂垹闄ょ儲寮曪紝灝嗙敓鎴愮殑*.del鏂囦歡鍒嗗彂鍒版墍鏈夌殑绱㈠紩鏈嶅姟鍣ㄤ笂鍘繪垨鑰呭HDFS绱㈠紩鐩綍鍒犻櫎绱㈠紩鍐嶅垎鍙戝埌瀵瑰簲鐨勭儲寮曟湇鍔″櫒涓婂幓銆?br /> b銆佹坊鍔?br /> 鏂版坊鍔犵殑鏁版嵁鐢ㄥ彟涓鍙版湇鍔″櫒鏉ョ敓鎴愩?br /> 鍒犻櫎鍜屾坊鍔犳楠ゅ彲鎸変笉鍚屽畾鏃剁瓥鐣ユ潵瀹炵幇銆?/p>
5銆丯ut鍒嗗竷寮忓茍琛岃綆楃壒鐐?br />
Nut鍒嗗竷寮忓茍琛岃綆楄櫧鐒朵篃鏄熀浜嶮/R妯″瀷錛屼絾鏄笌Hadoop M/R妯″瀷鏄笉鍚岀殑銆傚湪Hadoop M/R妯″瀷涓?Mapper鍜孯educer鏄竴涓畬鏁寸殑嫻佺▼錛孯educer渚濊禆浜嶮apper銆傛暟鎹簮閫氳繃Mapper鍒嗗彂鏈韓灝變細娑堣楀ぇ閲忕殑I/O錛屽茍涓旀槸娑堣桰/O鏈澶х殑閮ㄥ垎銆傛墍浠adoop M/R 騫跺彂鏄湁闄愮殑銆?br />
Nut M/R妯″瀷鏄皢Mapper鍜孯educer鍒嗙錛屽悇鑷嫭绔嬪瓨鍦ㄣ傚湪Nut涓?绱㈠紩浠ュ強绱㈠紩綆$悊 鏋勬垚M,鎼滅儲浠ュ強鎼滅儲鏈嶅姟鍣ㄧ粍 鏋勬垚 R銆?br />
浠ヤ竴涓垎綾葷粺璁℃潵璇存槑Nut鍒嗗竷寮忓茍琛岃綆楃殑嫻佺▼銆傚亣璁炬湁10涓垎綾伙紝瀵逛換鎰忓叧閿瘝鎼滅儲瑕佹眰緇熻鍑鴻鍏抽敭璇嶅湪榪?0涓垎綾諱腑鐨勬繪暟銆傚悓鏃跺亣璁炬湁10緇勬悳绱㈡湇鍔″櫒銆傜儲寮曚互鍙婄儲寮曠鐞嗚繘琛岀儲寮曟暟鎹殑Mapper錛岃繖鍧楁槸鍚庡彴鐙嚜榪愯綆$悊鐨勩侼ut Client灝嗚繖10涓垎綾葷粺璁″垎鍙戝埌10緇勬悳绱㈡湇鍔″櫒涓婏紝姣忕粍鎼滅儲鏈嶅姟鍣ㄥ鍏朵腑涓涓垎綾昏繘琛孯educer錛屽茍涓旀瘡緇勬悳绱㈡湇鍔″櫒鍙繘琛屽綰educer銆傛渶鍚庡皢鏈緇堢粨鏋滆繑鍥炵粰Nut Client銆?/p>
6銆佽璁″浘
7銆乑ookeeper鏈嶅姟鍣ㄧ姸鎬佺鐞嗙瓥鐣?/p>
鍦ㄦ灦鏋勮璁′笂閫氳繃浣跨敤澶氱粍鎼滅儲鏈嶅姟鍣ㄥ彲浠ユ敮鎸佹瘡縐掑鐞?00涓囦釜鎼滅儲璇鋒眰銆?br /> 姣忕粍鎼滅儲鏈嶅姟鍣ㄨ兘澶勭悊鐨勬悳绱㈣姹傛暟鍦?涓団?涓?鍗冧箣闂淬傚鏋滀嬌鐢?00緇勬悳绱㈡湇鍔″櫒錛岀悊璁轟笂姣忕鍙鐞?00涓囦釜鎼滅儲璇鋒眰銆?/p>
鍋囧姣忕粍鎼滅儲鏈嶅姟鍣ㄦ湁100浠界儲寮曟斁鍦?00鍙版鍦ㄨ繍琛屼腑鎼滅儲鏈嶅姟鍣?run)涓婏紝閭d箞灝嗙儲寮曟寜鐓у涓嬬殑鏂瑰紡鏀懼湪澶囩敤涓悳绱㈡湇鍔″櫒(bak)涓婏細index 1,index 2,index 3,index 4,index 5,index 6,index 7,index 8,index 9,index 10鏀懼湪B 1 涓婏紝index 6,index 7,index 8,index 9,index 10,index 11,index 12,index 13,index 14,index 15鏀懼湪B 2涓娿傘傘傘傘傘俰ndex 96,index 97,index 98,index 99,index 100,index 5,index 4,index 3,index 2,index 1鏀懼湪鏈鍚庝竴鍙板鐢ㄦ悳绱㈡湇鍔″櫒涓娿傞偅涔堟瘡浠界儲寮曚細瀛樺湪3鍙版満鍣ㄤ腑錛?浠芥鍦ㄨ繍琛屼腑錛?浠藉浠戒腑錛夈?br />
灝界榪欐牱璁捐姣忎喚绱㈠紩浼氬瓨鍦?鍙版満鍣ㄤ腑錛屼粛鐒朵笉鏄粷瀵瑰畨鍏ㄧ殑銆傚亣濡傝繍琛屼腑鐨刬ndex 1,index 2,index 3鍚屾椂瀹曟満鐨勮瘽錛岄偅涔堝氨浼氭湁涓浠界儲寮曟悳绱㈡湇鍔℃棤娉曟紜惎鐢ㄣ傝繖鏍瘋璁★紝浣滆呰涓烘槸鍦ㄥ畨鍏ㄦу拰鏈哄櫒璧勬簮涓よ呬箣闂翠竴涓瘮杈冮傚悎鐨勬柟妗堛?/p>
澶囩敤涓殑鎼滅儲鏈嶅姟鍣ㄤ細瀹氭椂媯鏌ヨ繍琛屼腑鎼滅儲鏈嶅姟鍣ㄧ殑鐘舵併備竴鏃﹀彂鐜頒笌鑷繁绱㈠紩瀵瑰簲鐨勬湇鍔″櫒瀹曟満灝變細鍚憀ock鐢寵鍒嗗竷寮忛攣錛屽緱鍒板垎甯冨紡閿佺殑鏈嶅姟鍣ㄥ氨灝嗚嚜宸卞姞鍏ュ埌榪愯涓悳绱㈡湇鍔″櫒緇勶紝鍚屾椂浠庡鐢ㄦ悳绱㈡湇鍔″櫒緇勪腑鍒犻櫎鑷繁錛屽茍鍋滄榪愯涓悳绱㈡湇鍔″櫒媯鏌ユ湇鍔°?/p>
涓鴻兘澶熸洿蹇熺殑寰楀埌鎼滅儲緇撴灉錛岃璁′笂灝嗘悳绱㈡湇鍔″櫒鍒嗕紭鍏堢瓑綰с傞氬父鏄皢鏈鏂扮殑鏁版嵁鏀懼湪涓鍙版垨鍑犲彴鍐呭瓨鎼滅儲鏈嶅姟鍣ㄤ笂銆傞氬父鎯呭喌涓嬪墠鍑犻〉鏁版嵁鑳藉湪榪欏嚑鍙版悳绱㈡湇鍔″櫒閲屾悳绱㈠埌銆傚鏋滃湪榪欏嚑鍙版悳绱㈡湇鍔″櫒涓婃病鏈夋暟鎹椂鍐嶅悜鍏朵粬鏃ф暟鎹悳绱㈡湇鍔″櫒涓婃悳绱€?br /> 浼樺厛鎼滅儲絳夌駭鐨勯昏緫鏄繖鏍風殑錛?鏈澶т負鎼滅儲鍏ㄩ儴鏈嶅姟鍣ㄥ茍涓?涓嶈兘浣滀負level鏍囪瘑銆傚綋鎼滅儲絳夌駭level涓?錛屾悳绱紭鍏堢駭涓?鐨勬湇鍔″櫒錛屽綋level涓?鏃舵悳绱紭鍏堢駭涓?鍜?鐨勬湇鍔″櫒錛屼緷姝ょ被鎺ㄣ?br />
浜屻侀厤緗?br />
1銆丩inux閰嶇疆
ssh-keygen –t rsa -P ''
cd .ssh
cp id_rsa.pub authorized_keys
/etc/hosts閲屽鍔?192.168.195.128 nz 騫朵笖灝?27.0.0.1 鏀逛負 192.168.195.128
2銆乭adoop閰嶇疆
hadoop-env.sh
JAVA_HOME=/home/nianzai/jdk1.6.0._22
core-site.xml
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/tmp/hadoop-nianzai</value>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://nz:9000</value>
</property>
</configuration>
hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>
mapred-site.xml
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>nz:9001</value>
</property>
</configuration>
sh hadoop namenode -format
sh start-all.sh
sh hadoop fs -mkdir input
3銆亃ookeeper閰嶇疆
zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/home/nianzai/zkdata
clientPort=2181
sh zkServer.sh start
4銆乭base閰嶇疆
hbase-env.sh
export JAVA_HOME=/home/nianzai/jdk1.6.0_22
export HBASE_MANAGES_ZK=false
hbase-site.xml
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://nz:9000/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.master.port</name>
<value>60000</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>nz</value>
</property>
</configuration>
regionservers
nz
sh start-hbase.sh
2銆佺壒鏂?br />
a銆佺儹鎻掓嫈
b銆佸彲鎵╁睍
c銆侀珮璐熻澆
d銆佹槗浣跨敤,涓庣幇鏈夐」鐩棤緙濋泦鎴?br />
e銆佹敮鎸佹帓搴?br />
f銆?*24鏈嶅姟
g銆佸け璐ヨ漿縐?
3銆佹悳绱㈡祦紼?br />
Nut鐢盜ndex銆丼earch銆丆lient銆丆ache鍜孌B浜旈儴鍒嗘瀯鎴愩?Cache榛樿浣跨敤memcached,DB榛樿浣跨敤hbase)
Client澶勭悊鐢ㄦ埛璇鋒眰鍜屽鎼滅儲緇撴灉鎺掑簭銆係earch瀵硅姹傝繘琛屾悳绱紝Search涓婂彧鏀劇儲寮曪紝鏁版嵁瀛樺偍鍦―B涓紝Nut灝嗙儲寮曞拰瀛樺偍鍒嗙銆侰ache緙撳瓨鐨勬槸鎼滅儲鏉′歡鍜岀粨鏋滄枃妗d銆侱B瀛樺偍鐫鏁版嵁錛孋lient鏍規嵁鎼滅儲鎺掑簭緇撴灉,鍙栧嚭褰撳墠欏典腑鐨勬枃妗d浠嶥B涓婅鍙栨暟鎹?/p>
鐢ㄦ埛鍙戣搗鎼滅儲璇鋒眰緇欑敱Nut Client鏋勬垚鐨勯泦緹わ紝鐢辨煇涓狽ut Client鏍規嵁鎼滅儲鏉′歡鏌ヨCache鏈嶅姟鍣ㄦ槸鍚︽湁璇ョ紦瀛橈紝濡傛灉鏈夌紦瀛樻牴鎹紦瀛樼殑鏂囨。id鐩存帴浠嶥B璇誨彇鏁版嵁錛屽鏋滄病鏈夌紦瀛樺皢闅忔満閫夋嫨涓緇勬悳绱㈡湇鍔″櫒緇?Search Group i),灝嗘煡璇㈡潯浠跺悓鏃跺彂緇欒緇勬悳绱㈡湇鍔″櫒緇勯噷鐨刵鍙版悳绱㈡湇鍔″櫒錛屾悳绱㈡湇鍔″櫒灝嗘悳绱㈢粨鏋滆繑鍥炵粰Nut Client鐢卞叾鎺掑簭錛屽彇鍑哄綋鍓嶉〉鏂囨。id錛屽皢鎼滅儲鏉′歡鍜屽綋鍓嶆枃妗d緙撳瓨錛屽悓鏃朵粠DB璇誨彇鏁版嵁銆?br />
4銆佺儲寮曟祦紼?br /> Hadoop Mapper/Reducer 寤虹珛绱㈠紩銆傚啀灝嗙儲寮曚粠HDFS鍒嗗彂鍒板悇涓儲寮曟湇鍔″櫒銆?br /> 瀵圭儲寮曠殑鏇存柊鍒嗕負涓ょ錛氬垹闄ゅ拰娣誨姞錛堟洿鏂板垎瑙d負鍒犻櫎鍜屾坊鍔狅級銆?br /> a銆佸垹闄?br /> 鍦℉DFS涓婂垹闄ょ儲寮曪紝灝嗙敓鎴愮殑*.del鏂囦歡鍒嗗彂鍒版墍鏈夌殑绱㈠紩鏈嶅姟鍣ㄤ笂鍘繪垨鑰呭HDFS绱㈠紩鐩綍鍒犻櫎绱㈠紩鍐嶅垎鍙戝埌瀵瑰簲鐨勭儲寮曟湇鍔″櫒涓婂幓銆?br /> b銆佹坊鍔?br /> 鏂版坊鍔犵殑鏁版嵁鐢ㄥ彟涓鍙版湇鍔″櫒鏉ョ敓鎴愩?br /> 鍒犻櫎鍜屾坊鍔犳楠ゅ彲鎸変笉鍚屽畾鏃剁瓥鐣ユ潵瀹炵幇銆?/p>
5銆乑ookeeper鏈嶅姟鍣ㄧ姸鎬佺鐞嗙瓥鐣?/p>
鍦ㄦ灦鏋勮璁′笂閫氳繃浣跨敤澶氱粍鎼滅儲鏈嶅姟鍣ㄥ彲浠ユ敮鎸佹瘡縐掑鐞?00涓囦釜鎼滅儲璇鋒眰銆?br /> 姣忕粍鎼滅儲鏈嶅姟鍣ㄨ兘澶勭悊鐨勬悳绱㈣姹傛暟鍦?涓団?涓?鍗冧箣闂淬傚鏋滀嬌鐢?00緇勬悳绱㈡湇鍔″櫒錛岀悊璁轟笂姣忕鍙鐞?00涓囦釜鎼滅儲璇鋒眰銆?/p>
鍋囧姣忕粍鎼滅儲鏈嶅姟鍣ㄦ湁100浠界儲寮曟斁鍦?00鍙版鍦ㄨ繍琛屼腑鎼滅儲鏈嶅姟鍣?run)涓婏紝閭d箞灝嗙儲寮曟寜鐓у涓嬬殑鏂瑰紡鏀懼湪澶囩敤涓悳绱㈡湇鍔″櫒(bak)涓婏細index 1,index 2,index 3,index 4,index 5,index 6,index 7,index 8,index 9,index 10鏀懼湪B 1 涓婏紝index 6,index 7,index 8,index 9,index 10,index 11,index 12,index 13,index 14,index 15鏀懼湪B 2涓娿傘傘傘傘傘俰ndex 96,index 97,index 98,index 99,index 100,index 5,index 4,index 3,index 2,index 1鏀懼湪鏈鍚庝竴鍙板鐢ㄦ悳绱㈡湇鍔″櫒涓娿傞偅涔堟瘡浠界儲寮曚細瀛樺湪3鍙版満鍣ㄤ腑錛?浠芥鍦ㄨ繍琛屼腑錛?浠藉浠戒腑錛夈?br />
灝界榪欐牱璁捐姣忎喚绱㈠紩浼氬瓨鍦?鍙版満鍣ㄤ腑錛屼粛鐒朵笉鏄粷瀵瑰畨鍏ㄧ殑銆傚亣濡傝繍琛屼腑鐨刬ndex 1,index 2,index 3鍚屾椂瀹曟満鐨勮瘽錛岄偅涔堝氨浼氭湁涓浠界儲寮曟悳绱㈡湇鍔℃棤娉曟紜惎鐢ㄣ傝繖鏍瘋璁★紝浣滆呰涓烘槸鍦ㄥ畨鍏ㄦу拰鏈哄櫒璧勬簮涓よ呬箣闂翠竴涓瘮杈冮傚悎鐨勬柟妗堛?/p>
澶囩敤涓殑鎼滅儲鏈嶅姟鍣ㄤ細瀹氭椂媯鏌ヨ繍琛屼腑鎼滅儲鏈嶅姟鍣ㄧ殑鐘舵併備竴鏃﹀彂鐜頒笌鑷繁绱㈠紩瀵瑰簲鐨勬湇鍔″櫒瀹曟満灝變細鍚憀ock鐢寵鍒嗗竷寮忛攣錛屽緱鍒板垎甯冨紡閿佺殑鏈嶅姟鍣ㄥ氨灝嗚嚜宸卞姞鍏ュ埌榪愯涓悳绱㈡湇鍔″櫒緇勶紝鍚屾椂浠庡鐢ㄦ悳绱㈡湇鍔″櫒緇勪腑鍒犻櫎鑷繁錛屽茍鍋滄榪愯涓悳绱㈡湇鍔″櫒媯鏌ユ湇鍔°?/p>
涓鴻兘澶熸洿蹇熺殑寰楀埌鎼滅儲緇撴灉錛岃璁′笂灝嗘悳绱㈡湇鍔″櫒鍒嗕紭鍏堢瓑綰с傞氬父鏄皢鏈鏂扮殑鏁版嵁鏀懼湪涓鍙版垨鍑犲彴鍐呭瓨鎼滅儲鏈嶅姟鍣ㄤ笂銆傞氬父鎯呭喌涓嬪墠鍑犻〉鏁版嵁鑳藉湪榪欏嚑鍙版悳绱㈡湇鍔″櫒閲屾悳绱㈠埌銆傚鏋滃湪榪欏嚑鍙版悳绱㈡湇鍔″櫒涓婃病鏈夋暟鎹椂鍐嶅悜鍏朵粬鏃ф暟鎹悳绱㈡湇鍔″櫒涓婃悳绱€?br /> 浼樺厛鎼滅儲絳夌駭鐨勯昏緫鏄繖鏍風殑錛?鏈澶т負鎼滅儲鍏ㄩ儴鏈嶅姟鍣ㄥ茍涓?涓嶈兘浣滀負level鏍囪瘑銆傚綋鎼滅儲絳夌駭level涓?錛屾悳绱紭鍏堢駭涓?鐨勬湇鍔″櫒錛屽綋level涓?鏃舵悳绱紭鍏堢駭涓?鍜?鐨勬湇鍔″櫒錛屼緷姝ょ被鎺ㄣ?br />
1銆佹榪?br /> 鍙負lucene鎻愪緵鍒嗗竷寮忔悳绱㈡鏋躲?*24鍗僄浠ヤ笂绱㈠紩鏂囦歡鏀寔鏁板崈涓囩駭鐨勭敤鎴鋒悳绱㈣闂?br /> Nut寮鍙戠幆澧冿細jdk1.6.0.21+lucene3.0.2+eclipse3.6+hadoop0.20.2+zookeeper3.3.1+linux
2銆佺壒鏂?br />
a銆佺儹鎻掓嫈
b銆佸彲鎵╁睍
c銆侀珮璐熻澆
d銆佹槗浣跨敤,涓庣幇鏈夐」鐩棤緙濋泦鎴?br />
e銆佹敮鎸佹帓搴?br />
f銆?*24鏈嶅姟
g銆佸け璐ヨ漿縐?
3銆佹悳绱㈡祦紼?br /> Nut鐢盜ndex銆丼earch銆丆lient銆丆ache鍜孌B浜旈儴鍒嗘瀯鎴愩?br /> Client澶勭悊鐢ㄦ埛璇鋒眰鍜屽鎼滅儲緇撴灉鎺掑簭銆係earch瀵硅姹傝繘琛屾悳绱紝Search涓婂彧鏀劇儲寮曪紝鏁版嵁瀛樺偍鍦―B涓紝Nut灝嗙儲寮曞拰瀛樺偍鍒嗙銆侰ache緙撳瓨鐨勬槸鎼滅儲鏉′歡鍜岀粨鏋滄枃妗d銆侱B瀛樺偍鐫鏁版嵁錛孋lient鏍規嵁鎼滅儲鎺掑簭緇撴灉,鍙栧嚭褰撳墠欏典腑鐨勬枃妗d浠嶥B涓婅鍙栨暟鎹?/p>
鐢ㄦ埛鍙戣搗鎼滅儲璇鋒眰緇欑敱Nut Client鏋勬垚鐨勯泦緹わ紝鐢辨煇涓狽ut Client鏍規嵁鎼滅儲鏉′歡鏌ヨCache鏈嶅姟鍣ㄦ槸鍚︽湁璇ョ紦瀛橈紝濡傛灉鏈夌紦瀛樻牴鎹紦瀛樼殑鏂囨。id鐩存帴浠嶥B璇誨彇鏁版嵁錛屽鏋滄病鏈夌紦瀛樺皢鏌ヨ鏉′歡鍚屾椂鍙戠粰鍚庨潰鐨刵鍙版悳绱㈡湇鍔″櫒錛屾悳绱㈡湇鍔″櫒灝嗘悳绱㈢粨鏋滆繑鍥炵粰Nut Client鐢卞叾鎺掑簭錛屽彇鍑哄綋鍓嶉〉鏂囨。id錛屽皢鎼滅儲鏉′歡鍜屽綋鍓嶆枃妗d緙撳瓨錛屽悓鏃朵粠DB璇誨彇鏁版嵁銆?/p>
4銆佺儲寮曟祦紼?br /> Hadoop Mapper/Reducer 寤虹珛绱㈠紩銆傚啀灝嗙儲寮曚粠HDFS鍒嗗彂鍒板悇涓儲寮曟湇鍔″櫒銆?br /> 瀵圭儲寮曠殑鏇存柊鍒嗕負涓ょ錛氬垹闄ゅ拰娣誨姞錛堟洿鏂板垎瑙d負鍒犻櫎鍜屾坊鍔狅級銆?br /> a銆佸垹闄?br /> 鍦℉DFS涓婂垹闄ょ儲寮曪紝灝嗙敓鎴愮殑*.del鏂囦歡鍒嗗彂鍒版墍鏈夌殑绱㈠紩鏈嶅姟鍣ㄤ笂鍘繪垨鑰呭HDFS绱㈠紩鐩綍鍒犻櫎绱㈠紩鍐嶅垎鍙戝埌瀵瑰簲鐨勭儲寮曟湇鍔″櫒涓婂幓銆?br /> b銆佹坊鍔?br /> 鏂版坊鍔犵殑鏁版嵁鐢ㄥ彟涓鍙版湇鍔″櫒鏉ョ敓鎴愩?br /> 鍒犻櫎鍜屾坊鍔犳楠ゅ彲鎸変笉鍚屽畾鏃剁瓥鐣ユ潵瀹炵幇銆?/p>
5銆乑ookeeper鏈嶅姟鍣ㄧ姸鎬佺鐞嗙瓥鐣?br />
鍋囧鎴戜滑鏈?00浠界儲寮曟斁鍦?00鍙版鍦ㄨ繍琛屼腑鎼滅儲鏈嶅姟鍣ㄤ笂錛岄偅涔堝皢绱㈠紩鎸夌収濡備笅鐨勬柟寮忔斁鍦ㄥ鐢ㄤ腑鎼滅儲鏈嶅姟鍣ㄤ笂錛歩ndex 1,index 2,index 3,index 4,index 5,index 6,index 7,index 8,index 9,index 10鏀懼湪B 1 涓婏紝index 6,index 7,index 8,index 9,index 10,index 11,index 12,index 13,index 14,index 15鏀懼湪B 2涓娿傘傘傘傘傘俰ndex 96,index 97,index 98,index 99,index 100,index 5,index 4,index 3,index 2,index 1鏀懼湪鏈鍚庝竴鍙板鐢ㄦ悳绱㈡湇鍔″櫒涓娿傞偅涔堟瘡浠界儲寮曚細瀛樺湪3鍙版満鍣ㄤ腑錛?浠芥鍦ㄨ繍琛屼腑錛?浠藉浠戒腑錛夈?br />
灝界榪欐牱璁捐姣忎喚绱㈠紩浼氬瓨鍦?鍙版満鍣ㄤ腑錛屼粛鐒朵笉鏄粷瀵瑰畨鍏ㄧ殑銆傚亣濡傝繍琛屼腑鐨刬ndex 1,index 2,index 3鍚屾椂瀹曟満鐨勮瘽錛岄偅涔堝氨浼氭湁涓浠界儲寮曟悳绱㈡湇鍔℃棤娉曟紜惎鐢ㄣ傞偅涔堣繖鏍瘋璁★紝浣滆呰涓烘槸鍦ㄥ畨鍏ㄦу拰鏈哄櫒璧勬簮涓よ呬箣闂翠竴涓瘮杈冮傚悎鐨勬柟妗堛?/p>
澶囩敤涓殑鎼滅儲鏈嶅姟鍣ㄤ細瀹氭椂媯鏌ヨ繍琛屼腑鎼滅儲鏈嶅姟鍣ㄧ殑鐘舵併備竴鏃﹀彂鐜頒笌鑷繁绱㈠紩瀵瑰簲鐨勬湇鍔″櫒瀹曟満灝變細鍏堝悜zookeeper鐢寵鍒嗗竷寮忛攣錛屽緱鍒伴攣鐨勬湇鍔″櫒灝卞皢鑷繁鍔犲叆鍒拌繍琛屼腑鎼滅儲鏈嶅姟鍣ㄧ粍錛屽悓鏃朵粠澶囩敤鎼滅儲鏈嶅姟鍣ㄧ粍涓垹闄よ嚜宸憋紝騫跺仠姝㈣繍琛屼腑鎼滅儲鏈嶅姟鍣ㄦ鏌ユ湇鍔°?br />