crowd - Brown University Department of Computer Science

!"#$%&#'"()*+,-../)(01#*&,0*%,
2/03#"4&5,-,6070,80*0+949*7,
29"&.9(1:9,
,
,
<=>
-*;0),6#0* =,8)(?09/,@A,B"0*C/)*D=,,,,,,,,,
6#*0/%,E#&&40**F=,G)4,E"0&C0D,,
,
<H*):9"&)7I,#J,K)&(#*&)*=,80%)&#*,
,>K0/40"7,L0M&,
DH*):9"&)7I,#J,!0/)J#"*)0=,N9"C9/9I,,
FOG;,P'")(?,
G'7#")0/,!#*79Q7,
•! R7?9",!"#$%&#'"()*+,G'7#")0/&5,
–! 2A,S.9)"#1&=,80*0+)*+,!"#$%&#'"(9%,;'40*,
!#4.'701#*=,KKK,T<<=,80"(?,>U<<A,
–! RA,-/#*&#=,8A,L90&9=,!"#$%&#'"()*+,J#",
S*J#"401#*,V97")9:0/5,2")*()./9&=,897?#%&=,0*%,
-../)(01#*&=,WSXSV,@'/I,>U<<A,
•! G?)&,7'7#")0/5,
–! O4.?0&)&,#*,?IM)"%,(#4.'79"Y?'40*,&I&794&,J#",
Z'9"I,."#(9&&)*+A,
–! S*79+"01#*,$)7?,7"0%)1#*0/,%070M0&9,4#%9/&=,
0/+#")7?4&,0*%,&I&794&,0"(?)79(7'"9&A,
N)//)#*&,#J,69:)(9&,
HM)Z')7#'&,!#**9(1:)7I,
F,,
!"#$%&#'"!()*+,-./01,&+
!)1[9*,W()9*(9,
\-W-,]!/)(C$#"C9"&^,()"(0,>UUU,
!)1[9*,@#'"*0/)&4,0*%,20"1().07#"I,
W9*&)*+,
_,
6N,&.9()`(,
•! B"99M0&9,
&#+$2.3+(&+(34+
R*9,a)9$,#J,!"#$%&#'"()*+,
B"#4,b')**,c,N9%9"&#*=,];'40*,!#4.'701#*5,-,W'":9I,0*%,G0Q#*#4I,#J,0,
X"#$)*+,B)9/%^=,!;S,>U<<A,
G?9,K0I,S*%'&7"I,L##C&,-7,S7,
G0Q#*#4)9&,
,
•! 6#0*=,;0/9:I=,V040C")&?*0*d,e!"#$%&#'"()*+f,,
!-!8,FY<<,
–! *07'"9,#J,(#//0M#"01#*,e)4./)()7,:&A,9Q./)()7f,
–! 0"(?)79(7'"9,e&70*%0/#*9,:&A,.)++IM0(C9%f,
–! 4'&7,"9("')7,'&9"&Y$#"C9"&g,,eI9&,#",*#f,
–! K?07,%#,'&9"&Y$#"C9"&,%#g,e,
•! N9%9"&#*,c,b')**d,e;'40*,!#4.'701#*f,!;S,h<<,
–! 8#1:01#*,e20I=,-/7"')&4=,O*i#I49*7=,V9.'701#*f,
–! b'0/)7I,!#*7"#/,e49(?0*)&4&,j,&99,G)4h&,&9(1#*f,
–! -++"9+01#*,e?#$,0"9,"9&'/7&,(#4M)*9%gf,
–! ;'40*,WC)//,ea)&'0/,"9(#+*)1#*=,/0*+'0+9=,kf,
–! k,
20"1().07#"I,!'/7'"9,l,OQ./)()7,
<F,,
20"1().07#"I,!'/7'"9,j,S4./)()7,
@#?*,8'""9//5,X8,Wa,nY<_YUn,
k9:9"I,149,$9,'&9,0,X##+/9,0..,#",&9":)(9=,$9,0"9,
$#"C)*+,#*,M9?0/J,#J,7?9,&90"(?,&#:9"9)+*=,("901*+,
4#"9,(#*79*7,J#",)7,7#,)*%9Q,0*%,4#*91[9,#",790(?)*+,
)7,&#497?)*+,.#79*10//I,'&9J'/,0M#'7,#'",%9&)"9&=,
)*79*1#*&,0*%,M9?0:)#"A,
,
,
<m,,
GI.9&,#J,G0&C&,
3567+*859:;58<=>+
,?5@A;B6+
!#4./9Q,G0&C&,
•! N')/%,0,$9M&)79,
•! 69:9/#.,0,&#o$0"9,&I&794,
•! R:9"7?"#$,0,+#:9"*49*7g,
W)4./9,2"#i9(7&,
•! 69&)+*,0,/#+#,0*%,:)&'0/,)%9*17I,
•! K")79,0,79"4,.0.9",
80("#,G0&C&,
•! K")79,0,"9&70'"0*7,"9:)9$,
•! G9&7,0,*9$,$9M&)79,J907'"9,
•! S%9*1JI,0,+0/0QI,
8)("#,G0&C&,
•! L0M9/,0*,)40+9,
•! a9")JI,0*,0%%"9&&,
•! W)4./9,9*17I,"9&#/'1#*,
S*&.)"9%,MI,7?9,"9.#"75,]20)%,!"#$%&#'"()*+^=,W40"7&?997A(#4=,nY<mY>UUn,
/(!"#C3.&D+/."D,301.!,&+
-40[#*,89(?0*)(0/,G'"C,e-8Gf,
8)("#70&C)*+,j,a)"'70/)[9%,;'40*&,
•! !'""9*7,/90%9"5,-40[#*,89(?0*)(0/,G'"C,
•! V9Z'9&7#"&,./0(9,;'40*,S*79//)+9*(9,G0&C&,e;SG&f,
–! 8)*)4'4,.")(95,pUAU<,
–! q#J,"9./)(0&,e0&&)+*49*7&f=,9Q.)"01#*=,'6B8+(9=B8E5FB+
–! -2SlM0&9%5,]("9079;)7ef^=,]+97-&&)+*49*7&ef^=,
]0.."#:9-&&)+*49*7&ef^=,]J#"(9OQ.)"9ef^,
–! V9Z'9&7#"&,0.."#:9,i#M&,0*%,.0I49*7,
•! K#"C9"&,e0ACA0A,]7'"C9"&^f,(?##&9,i#M&=,%#,7?94=,+97,
.0)%,
<n,
8)("#70&C,-++"9+07#"&,
W040&#'"(9A#"+,
2GH"(%+2'/.)I!#/0'3,"+
!#/0'3.3(#)+
\#7,OQ0(7/I,!"#$%&#'"()*+=,M'7k,
]G?9,?#.9,)&,7?07=,)*,*#7,7##,40*I,I90"&=,?'40*,M"0)*&,0*%,(#4.'1*+,
40(?)*9&,$)//,M9,(#'./9%,7#+97?9",:9"I,1+?7/I=,0*%,7?07,7?9,"9&'/1*+,
.0"7*9"&?).,$)//,7?)*C,0&,*#,?'40*,M"0)*,?0&,9:9",7?#'+?7,0*%,."#(9&&,%070,)*,0,
$0I,*#7,0.."#0(?9%,MI,7?9,)*J#"401#*l?0*%/)*+,40(?)*9&,$9,C*#$,7#%0IA^,
G?)*C)*+,-M#'7,;IM")%,WI&794&,
-/+#")7?4&,
K07&#*YSN8,
&90"(?,
80(?)*9&,
29#./9,
>_,
OQ04./95,;IM")%,S40+9,W90"(?,
s0*=,E'40"=,X0*9&0*=,!"#$%W90"(?5,OQ./#)1*+,!"#$%&,J#",-(('"079,V90/l149,S40+9,
W90"(?,#*,8#M)/9,2?#*9&=,8#M)&I&,>U<UA,,
>r,
OQ04./95,;IM")%,6070,S*79+"01#*,
paper
conf
title
author
email
venue
Data integration
VLDB-01
OLAP
Mike
mike@a
ICDE-02
Data mining
SIGMOD-02
Social media
Jane
jane@b
PODS-05
!!
+*B9B85=B+A;5:6<J;B+@5=FKB6+
j! .0.9",t,17/9=,.0.9",t,0'7?#"=,.0.9",t,940)/=,.0.9",t,:9*'9,
j! (#*J,t,17/9=,(#*J,t,0'7?#"=,(#*J,t,940)/=,(#*J,t,:9*'9,
!!
.67+:6B86+=L+MB8<E>++
6#9&,0u")M'79,.0.9",407(?,0u")M'79,0'7?#"g,,
paper
conf
title
author
email
Data integration
VLDB-01
OLAP
Mike
mike@a
Data mining
SIGMOD-02
Social media
Jane
jane@b
s9&,
\#,
\#7,&'"9,
8(!0**=,W?9*=,6#0*5,807(?)*+,W(?940&,)*,R*/)*9,!#44'*)19&A,S!6O=,>UUr,
OQ04./95,;IM")%,b'9"I,2"#(9&&)*+,,
N'7,*#75,
•! -*I7?)*+,7?9,(#4.'79",
0/"90%I,%#9&,$9//,,
MetaData
,
K?9"9,7#,'&9,7?9,("#$%5,
•! Q<9O+@<66<9R+O5=5+
•! /57B+6:JSBFTMB+FL@A58<6L96+
•! "BFLR9<UB+A5VB896,
CrowdSQL
Statistics
'6B+=KB+F8LNO+=L+
596NB8+%HCK58O+
P:B8<B6!
Results
Parser
Turker Relationship
Manager
Optimizer
UI
Creation
Form
Editor
Executor
UI Template Manager
Files Access Methods
HIT Manager
Disk 1
Disk 2
8A,B"0*C/)*=,6A,E#&&40**=,GA,E"0&C0=,WA,V049&?,0*%,VA,v)*,!"
"!"#$%6N5,-*&$9")*+,b'9")9&,$)7?,!"#$%&#'"()*+=,#$%&'(")*++""
DU,
G?9,V9&7,#J,7?9,G'7#")0/,
•! W04./9,]"9/01#*0/^,%070,40*0+949*7,0../)(01#*&,
–! )*J#"401#*,9Q7"0(1#*,
–! &(?940,407(?)*+,
•! !"#$%&#'"()*+,6N,./03#"4&,
–! %070,4#%9/&=,Z'9"I,/0*+'0+9&,
–! Z'9"I,."#(9&&)*+=,#.14)[01#*,
–! Z'0/)7I,)&&'9&,
•! W04./9,]M9I#*%l"9/01#*0/^,%070,40*0+949*7,
0../)(01#*&,
–! +"0.?,&90"(?,Y,(/0&&)`(01#*,
–! 4#M)/9,)40+9,&90"(?,
–! &#()0/,49%)0,0*0/I&)&,eK0/40"7L0M&f,
•! V9&90"(?,(?0//9*+9&,
D<,
Crowdsourced Data Management
Applications
!!
Relational
–! information extraction
–! schema matching
–! entity resolution
–! data spaces
–! building structured KBs
–! sorting
–! top-k
–! ...
!!
Beyond relational
–! graph search
–! classification
–! mobile image search
–! social media analysis
–! question answering
–! NLP
–! text summarization
–! sentiment analysis
–! semantic wikis
–! ...
(Apologies if we forget your favorite applications)
32
Key Issues
!!
The role of machine (i.e., algorithm) and humans
–! use only humans? both? who’s doing what?
!!
Recruiting models
–! pay? volunteer?
!!
!!
!!
User interfaces
Quality control
Optimization
33
Information Extraction
Amplifying Community Content Creation with MixedInitiative Information Extraction. R. Hoffman, S. Amershi,
J. Fogarty, K. Patel, D. Weld, F. Wu. CHI-2009
34
Problem: Populate Infoboxes
35
Solution: IE Using Machine + Human
Hemingway was an American author ...
Infobox
The American readers ...
Born
July 21, 1899
Nationality American
Train a “nationality” extractor
Apply extractor to new pages
to extract nationalities.
Raybradbury is a well-known
American author ...
" Nationality = American
Christa Wolf is a German author ...
" Nationality = German
Verify with crowd
36
Recruiting Model: Search Advertising
“ray bradbury”
!!
Interrupt user in the middle of a primary task
–! searching for information on Ray Bradbury
!!
Ask if user is willing to contribute
!!
Evaluate three UIs: pop-up, highlight, icon
–! in terms of intrusiveness and willingness to contribute
37
!"#$#%&'()*+,-)%
er
hov
./012/01(%&'()*+,-)%
./012/01(%&'()*+,-)%
er
hov
./012/01(%&'()*+,-)%
./012/01(%&'()*+,-)%
er
hov
&-"'%&'()*+,-)%
&-"'%&'()*+,-)%
er
hov
&-"'%&'()*+,-)%
&-"'%&'()*+,-)%
!"#$%&'(&)&$)*+&)&,+)-$%./(0)1/0,&)
2-+)&,/(3)1'4)5%'.67%489)('&/$(':/&4)
/9);"+%/<'(=)!9)&,/9)<$%%+<&>?)
2@:+'9+)<,+<3)A/&,)&,+)5%/&'((/<'B?)
2-+)&,/(3)&,+)97""'%4)9,$7:.)9'4)1'4)
5%'.67%489)('&/$(':/&4)/9);"+%/<'(=)!9)&,/9)
2!C)!)3(+A)A$7:.)!)%+'::4)(++.)&$):$$3?)
A,'&)&,+)'%&/<:+)9'49>?)
Experimental Results
!!
Deployed with Yahoo / Google
–!
–!
–!
–!
7 days
$1500
2473 visitors
224 answers
!!
Icon < Highlight < Popup in intrusiveness
Icon < Highlight < Popup in willingness to contribute
!!
Out of 224 answers
!!
–! 90% marked correct are indeed correct
–! 58% marked incorrect are indeed incorrect
!!
Training on noisy answers improves extraction accuracy
48
Schema Matching
Matching Schemas in Online Communities: A Web 2.0
Approach. R. McCann, W. Shen, A. Doan. ICDE-08
49
Schema Matching
paper
conf
title
author
email
venue
Data integration
VLDB-01
OLAP
Mike
mike@a
ICDE-02
Social media
Jane
jane@b
PODS-05
Data mining
!!
SIGMOD-02
Generate plausible matches & ask users to verify
–! paper = title, paper = author, paper = email, paper = venue
–! conf = title, conf = author, conf = email, conf = venue
Does attribute paper match attribute author?
paper
conf
title
author
email
Data integration
VLDB-01
OLAP
Mike
mike@a
Social media
Jane
jane@b
Data mining
SIGMOD-02
Yes
No
Not sure
Recruiting Model
!!
Make users pay for service
paper = author?
51
Quality Control
!!
Classify users into trusted/untrusted
–! if (U has correctly answered X out of Y evaluation questions)
AND (Y >= t1) AND (X/Y >= t2) " U is trusted
!!
Monitor trusted answers to question Q. Stop when
–! at least t3 answers
–! gap between yes/no answers is at least t4
–! if gap has not been reached by t5 answers, stop with majority voting
!!
Example
–! minimum # of answers t3 = 6, gap t4 = 3, maximum t5 = 9
paper = author?
Yes, No, No, Yes, Yes, Yes, Yes " Yes
Yes, Yes, Yes, No, Yes, No, No, No, No " No
!!
Must continuously monitor and re-valuate trusted users
52
Optimization
!!
Exploit constraints paper = title
paper = author
paper = email
paper = venue
!!
!!
conf = title
conf = author
conf = email
conf = venue
Use algorithm to re-rank lists & remove certain matches
Round robin
paper = title, .8
paper = author, .6
paper = email, .3
Q1
Q2
Q3
Q4
Q5
Q6
conf = author, .7
conf = venue, .6
conf = email, .4
conf = title, .1
If “human oracle” is correct with prob 0.95
" prob of correctly answering Q6 = 0.77
53
!"#$%l&#'"()*+,6070M0&9,WI&794&,
,
G$#,C)*%&,#J,&I&794&,
•! W.9()`(,0../)(01#*&,7?07,40C9,'&9,#J,("#$%&,
–! ("#$%l&#'"(9,0,.0"1('/0",C)*%,#J,)*J#"401#*,
–! 9A+A=,0,.0"1('/0",;!S,9Q.9")49*7,#*,-8G,
•! 2/03#"4&,7?07,9*0M/9,0..&,$)7?,("#$%&,
–! 0'7#4079,(9"70)*,70&C&,7?07,)*:#/:9,("#$%&,
–! ."#:)%9,?)+?9"l/9:9/,&9":)(9&,7?0*,-8G,97,0/A,
–! $9,4#&7/I,)*79"9&79%,)*,]%070M0&9,&9":)(9&^,
6070M0&9&,0*%,!"#$%&,
•! ;#$,(0*,("#$%&,?9/.,%070M0&9&g,
–! `Q,M"#C9*,%0705,9*17I,"9&#/'1#*=,)*(#*&)&79*()9&=,k,
–! 0%%,4)&&)*+,%070,
–! &'Mi9(1:9,(#4.0")&)#*&,
•! ;#$,(0*,%070M0&9&,?9/.,("#$%,0..&g,
–! /0[I,%070,0(Z')&)1#*5,#*/I,+97,%070,7?07,)&,*9(9&&0"I,
–! +049,7?9,$#"C9"&,40"C97,
–! &94)l0'7#401(0//I,("9079,'&9",)*79"J0(9&,
–! 40*0+9,7?9,%070,&#'"(9%,J"#4,7?9,("#$%,
•! .;;+6>6=B@6+O<6F:66BO+KB8B+FLMB8+H#32+6<OB6++
X9*9")(,-"(?)79(7'"9,
0../)(01#*,
Z'9")9&,
e'.%079&f,
0*&$9"&,
2/03#"4,
+97,Y,.'7,
70&C&,
OQ04./9,WI&794&,
•!
•!
•!
•!
•!
•!
•!
!"#$%6N,eN9"C9/9I=,OG;f,
b'"C,,e8SGf,
W(##.,eW70*J#"%f,
;/#+,,eK)&(#*&)*f,
B"99M0&9,eX##+/9f,
B'&)#*!R82,
-.#/#+)9&,)J,$9,J#"+#7,I#'",J0:#")79,&I&794,
–! 6)&(/0)49"5,M)0&,7#,!"#$%!"#$%,4991*+,ewY>U<<f,
R:9":)9$,#J,G#.)(&,!#:9"9%,
•! 6070,8#%9/,0*%,b'9"I,L0*+'0+9,
,
•! b'9"I,2"#(9&&)*+,G9(?*)Z'9&,
,
•! b'0/)7I,!#*7"#/,
•! ,-./012"/3454.067"89/8":6";/1148";4<6="/55"/736;87!>"
6070,8#%9/5,W#49,N)+,b'9&1#*&,
•! b<5,W049,4#%9/,J#",9/9(7"#*)(,0*%,("#$%,%070g,
–! s9&=,)*,0//,&I&794&,$9,0"9,0$0"9,#J,
•! b>5,H&9",:)9$,:&A,&I&794,:)9$,#*,%070g,
–! 6)x9"9*7,)*,4#&7,&I&794& ,eJ#",:0")#'&,"90&#*&f,
•! bD5,R.9*,:&A,(/#&9%,$#"/%g,,
,,,,,,,,;#$,7#,M#'*%,?'40*,)*.'7,"9Z')"9%,J#",0,Z'9"Ig,
–! (/#&9%,$#"/%5,%070M0&9,%979"4)*9&,"9Z')"9%,)*.'7,
–! #.9*,$#"/%,<5,/)4)7,Z'9")9&,7?07,I#',40I,0&C,
–! #.9*,$#"/%,>5,("#$%,ey&I&794f,0*&$9",7?)&,Z'9&1#*=,7##,,
6070,8#%9/,c,L0*+'0+95,,B)*9,2")*7,
•!
•!
•!
•!
•!
•!
,
;#$,7#,%90/,$)7?,'*(9"70)*7Ig,
W'..#"7,J#",/)*90+9g,
K?9*,)&,%070,("#$%l&#'"(9%g,
;#$,7#,&.9()JI,?#$,7#,+97,%070g,
H&9",:)9$,:&A,&I&794,:)9$,#*,%070g,
W'..#"7,J#",."#+"04401(,'.%079&g,
b'"C,e8SGf,
•! X#0/5,("#$%l&#'"(9,(#4.0")&#*&=,4)&&)*+,%070,
•! N0&)&5,WbLD,y,H6B,
–! H6B,9*(0.&'/079,("#$%,)*.'7,
–! &.9()0/,794./079,/0*+'0+9,J#",("#$%,H6B&,
–! &.9()JI,HS=,Z'0/)7I,(#*7"#/=,k,e.#&&)M/I,#.7A,?)*7&f,
•! V9J9"9*(9&,
–! z80"('&,97,0/A,!S6V,>U<<{,
–! z80"('&,97,0/A,,WSX8R6,>U<<{,
–! z80"('&,97,0/A,,!"#$%!"#$%,>U<<{,
b'"C,OQ04./9,z80"C'&,97,0/A,!"#$%!"#$%,>U<<{,
•! G0&C5,B)*%,0//,$#49*,)*,0,].9#./9^,%070M0&9,
•! W(?940,
CREATE TABLE people(
name varchar(256),
photo blob );
•! b'9"I,
SELECT name
FROM people p
WHERE isFemale(p);
b'"C,OQ04./9,z80"C'&,97,0/A,!"#$%!"#$%,>U<<{,
•! G0&C5,B)*%,0//,$#49*,)*,0,].9#./9^,%070M0&9,
•! W(?940,
CREATE TABLE people(
name varchar(256),
photo blob );
3.&D+<6QB@5;BW=:A;BX+3G0,Y+Q<;=B8+
•! b'9"I,
+Z:B6TL9Y++[<6+\6+QB@5;B]^++
SELECT name
+ + + +++++++++=:A;B_[AKL=L]`+
FROM people p
+GB63B?=Y++++[GB6]+
WHERE isFemale(p); +)L3B?=Y+++++[)L]+
b'"C,OQ04./9,z80"C'&,97,0/A,!"#$%!"#$%,>U<<{,
•! G0&C5,B)*%,0//,$#49*,)*,0,].9#./9^,%070M0&9,
•! W(?940,
CREATE TABLE people(
name varchar(256),
photo blob );
3.&D+<6QB@5;BW=:A;BX+3G0,Y+Q<;=B8+
•! b'9"I,
+Z:B6TL9Y++[<6+\6+QB@5;B]^++
SELECT name
+ + + +++++++++=:A;B_[AKL=L]`+
FROM people p
+GB63B?=Y++++[GB6]+
WHERE isFemale(p); +)L3B?=Y+++++[)L]+
G?9,40+)(,)&,)*,7?9,G94./079&,
•! G94./079&,+9*9"079,HS&,J#",%)x9"9*7,C)*%&,#J,
("#$%l&#'"()*+,70&C&,
–! `/79"&5,,s9&,Y,\#,Z'9&1#*&,
–! i#)*&5,,,(#4.0")&#*&,M97$99*,7$#,7'./9&,e9Z'0/)7If,
–! #"%9",MI5,(#4.0")&)#*&,M97$99*,7$#,7'./9&,e+7gf,
–! +9*9"01:95,("#$%l&#'"(9,0u")M'79,:0/'9&,
•! G94./079&,0/&#,&.9()JI,Z'0/)7I,(#*7"#/d,9A+A=,
!#/H(),"Y++/5SL8<=>aL=B+
!"#$%6N,eN9"C9/9I=,OG;f,
•! X#0/5,("#$%l&#'"(9,(#4.0")&#*&=,4)&&)*+,%070,
–! e0/4#&7f,7?9,&049,0&,b'"C,
–! %)x9"9*(95,("#$%l&#'"(9,*9$,7'./9&,
•! 69&)+*5,,,WbL,$)7?,&/)+?7,9Q79*&)#*&,7#,68L=,bL,
–! !VRK6,C9I$#"%,7#,)*%)(079,("#$%l&#'"(9%,%070,
–! &I*70(1(,&'+0",J#",("#$%l&#'"(9%,(#4.0")&#*&,
•! V9J9"9*(9&,
–! zB"0*C/)*,97,0/A,WSX8R6,>U<<{,,
–! zv)*,97,0/A,!"#$%!"#$%,>U<<{,
–! zB9*+,97,0/A,aL6N,>U<<{,
!"#$%WbL,
•! !VRK6,(#/'4*&,
–! 9*119&,C*#$*=,."#.9"19&,#J,9*119&,40I,M9,'*C*#$*,
,CREATE TABLE company (
name STRING PRIMARY KEY,
hq_address CROWD STRING);
•! !VRK6,70M/9,
–! 9*119&,'*C*#$*=,("#$%l&#'"(9,*9$,9*119&,
,CREATE CROWD TABLE department (
university STRING,
department STRING,
phone_no STRING)
PRIMARY KEY(university, department) );
,
!"#$%WbL,W940*1(&,
•! INSERT INTO company(name) VALUES(“Apple”);
–! &97,9?@/AA=677,7#,0,&.9()0/,("#$%l&#'"(9,'*C*#$*,
–! e0/&#,.#&&)M/9,7#,&97,9?@/AA=677,0&,.0"7,#J,S\WOVGA,
•! SELECT * FROM company;
–! "97'"*,0//,(#4.0*)9&,$)7?,1/B6,0*%,9?@/AA=677"
–! )J,9?@/AA=677,)&,&.9()0/,'*C*#$*=,7?9*,0&C,("#$%,,
–! &7#"9,("#$%l&#'"(9,9?@/AA=677,)*,%070M0&9,,
•! )A9A=,'.%079,0&,0,&)%9l9x9(7,#J,0,Z'9"I|||,,
!"#$%WbL,W940*1(&,e(7%Af,
•! X):9,49,<U,'*):9")&)19&,7?07,?0:9,0,!W,%9.7A,
SELECT university FROM department
WHERE department = “Computer Science”
LIMIT 10;
•! G?)&,Z'9"I,)&,9Q9('79%,0&,J#//#$&k,
–! 9:0/'079,0//,%9.0"749*7&,0/"90%I,"9(#"%9%,)*,7?9,6N,
•! .#&&)M/I,("#$%l&#'"(9,7?9,A63/=8B618=,C10<6=708D,0u"&A,
•! e0(7'0//I=,*#7,*99%9%,)*,7?)&,(0&9,M9(0'&9,.0"7,#J,7?9,C9If,
–! )J,*#7,&'}()9*7=,0&C,7?9,("#$%,J#",0%%)1#*0/,%9.7&A,
–! &7#"9,*9$/I,("#$%l&#'"(9%,%9.7&A,)*,7?9,%070M0&9,
–! e-(7'0//I=,40*I,0/79"*01:9,$0I&,7#,%#,7?)&,l~,&99,/079"f,
69(#,eW70*J#"%=,W(##.,2"#i9(7f,
•! X#0/5,,("#$%l&#'"(9,4)&&)*+,%070,
–! J#('&,#*,%070,4#%9/,,e(#4.0")&#*&,0"9,#"7?#+#*0/f,
–! +9*9"0/)[9&,y,J#'*%01#*&,J#",("#$%l&#'"(9%,4#%9/&,
•! 69&)+*5,,"9/01#*0/,%070,4#%9/,
–! (/90*,&9.0"01#*,M97$99*,'&9",y,&I&794,:)9$,
–! J97(?,"'/9&5,,0%%,("#$%l&#'"(9,7'./9&,c,0u")M'79&,
–! "9&#/'1#*,"'/9&5,,"9&#/:9,(#*)(7&,M97$99*,$#"C9"&,
•! V9J9"9*(9&,
–! zK)%#4,97,0/A,!"#$%!"#$%,>U<<{,
69(#5,H&9",:&A,WI&794,a)9$,
•! W(?940,69&)+*9",
–! %9`*9&,(#*(9.7'0/=,"9/01#*0/,&(?940,e0&,'&'0/f,
–! %9`*9&,J'*(1#*0/,%9.9*%9*()9&,eÄ,0&,'&'0/f,
–! %9`*9&,J97(?,0*%,"9&#/'1#*,"'/9&,e*9$f,
•! H&9",
–! &99&,(#*(9.7'0/=,"9/01#*0/,&(?940,0*%,$")79&,0..,,
•! WI&794,
–! %9`*9&,&I&794,e)*79"*0/f,&(?940,7#,&7#"9,%070,
–! (#*(9.7'0/,&(?940,)&,0,:)9$,#*,)*79"*0/,&(?940,
L0I9"&,#J,69(#,6070,8#%9/,
'&9",
%9&)+*9",
!#*(9.7'0/,W(?940,
&I&794,
S*79"*0/,W(?940,
6070M0&9,
("#$%,
69(#,e&)4./)`9%f,OQ04./9,zK)%#4,>U<<{,
•! !#*(9.7'0/,&(?940,e'&9",.9"&.9(1:9f,
–! Restaurant(name, [rating], [cuisine])
•! B97(?,Y,V9&#/'1#*,"'/9&,
–! name -> rating
–! name -> cuisine
–! Ø -> name
(resolution with Method “AVG”)
(resolution with Method “DUPL-ELIM”)
–! =/E1.2";C07016"0"9,%9.9*%9*7,0u")M'79&,
–! 1/B6")&,0*(?#",e#o9*=,M'7,*#7,*9(9&&0")/I,C9If,
•! S*79"*0/,&(?940,
–! #*9,70M/9,J#",90(?,0*(?#",
–! #*9,70M/9,J#",90(?,J97(?,"'/9,7#,"9&#/:9,(#*)(7&,
69(#,OQ04./9,
95@B+
95@B+
85T9R+
95@B+
F:<6<9B+
-*7#*h&,
-*7#*h&,
m,
-*7#*h&,
B"9*(?,
-*7#*h&,
D,
-*7#*h&,
!#*1*9*7,
8(6,
D,
8(6,
B0&7B##%,
k,
k,
k,
k,
8(6#*0/%,
k,
)*79"*0/,:)9$,
'&9",:)9$,
95@B+
85T9R+
F:<6<9B+
-*7#*h&,
F,
B"9*(?,
-*7#*h&,
F,
!#*1*9*70/,
8(6#*0/%,
D,
B0&7B##%,
;/#+,eK)&(#*&)*f,
•! X#0/5,,W.9()JI,$#"C#$&,$)7?,?'40*,)*.'7,
–! 4079")0/)[9,?'40*,)*.'7,J#","9'&9,
•! 69&)+*5,,M0&9%,#*,6070/#+,
–! %070/#+,"'/9&,%9`*9,`*0/,+#0/,#J,$#"C#$,
–! %070/#+,"'/9&,%9`*9,&79.&,7#,"90(?,7?07,+#0/,
–! %070/#+,"'/9&,%9`*9,("#$%l&#'"(9%,)*.'7,
•! V9J9"9*(9&,
–! z!?9*,97,0/A,aL6N,>UU_{,,
–! z!?0),97,0/A,WSX8R6,>UUn{,
;/#+,OQ04./9,z6#0*,>U<<{,
•! OQ7"0(7,2!,(?0)"&,#J,(#*J9"9*(9&,J"#4,K9M,.0+9,
–! ("0$/,.0+9&5,,,
•! .0+9&e.0+9f,5l,%070W#'"(9&eHVL=,.0+9f=,("0$/eHVL=,.0+9f,
–! 9Q7"0(7,(#*J9"9*(9&5,
•! (#*J9"9*(9&e(=,.f,5l,.0+9&e.f=,9Q7"0(7!#*Je.=,(f,
–! 9Q7"0(7,.9#./95,
•! *049&e*=,.f,5,.0+9&e.f=,9Q7"0(7\049e.=,*f,
–! 9Q7"0(7,"#/9&,e9A+A=,2!,!?0)"5,
•! "#/9e*=,"=,.f,5l,*049&e*=,.f=,9Q7"0(7V#/9e*=,"=,.f,
–! .'Å*+,)7,7#+97?9"5,,
•! &9":)(9e*=,"=,(f,5l,(#*J9"9*(9&e(=,.f=,"#/9e*=,"=,.f,
•! e*#,("#$%l&#'"()*+,I97kf,
;/#+,$)7?,!"#$%lW#'"()*+,
•! -&C,.9#./9,7#,9Q7"0(7,7?9,"#/9&,J"#4,.0+9&,
–! CSroles(n, r, p#no-edit)#spreadheet-UI :- role(n, r, p)
–! 20+9,.,)&,0*,)*.'7,
–! H&9",("9079&,&."90%&?997,#J,e*=,"=,.f,7'./9&,
•! R7?9",."0+40&,J#",#7?9",.'".#&9&,
–! 9A+A=,qJ#"4lHS=,q$)C)lHS,
R7?9",;/#+,B907'"9&,
•! V9'&9,#J,("#$%l&#'"(9%,)*.'7,
–! '&9",%9`*9%,7'./9&,0"9,40)*70)*9%,)*,%070M0&9,
–! %070/#+,"'/9&,(0*,M9,&99*,0&,e4079")0/)[9%f,:)9$&,
–! C99.,/)*90+9,)*,#"%9",7#,%)&1*+')&?,M97$99*,("#$%l
&#'"(9%,0*%,(#4.'79%,)*.'7,J#",:)9$&,
–! e&)4)/0",0.."#0(?,7#,!"#$%6N,0*%,69(#f,
•! K?9*,7#,("#$%l&#'"(9,
–! ;/#+,&'..#"7&,]%"):9lMI,("#$%l&#'"()*+^,
•! +97,)*J#"401#*,J"#4,$#"C9"&,$?9*,$#"C9"&,0"9,"90%I,
•! *#7,*9(9&&0"I,$?9*,0..,)&,"90%I,7#,(#*&'49,)*J#"401#*,
–! e*#7,(#*&)%9"9%,)*,b'"C=,!"#$%6N=,69(#f,
B"99M0&9,Y,8970K9M,eX##+/9f,
•! X#0/5,69&(")M9,7?9,$?#/9,$#"/%,
–! >>,4)//)#*,9*119&,0*%,7?9)","9/01#*&?).&,
–! 9*17I,t,.9"&#*,Ç,./0(9,Ç,7?)*+,
–! 9*17I,"9&#/'1#*,$)7?,&)*+/9,K9MS6,eÄ,M0"(#%9=,HVSf,
•! B#'*%01#*5,,W940*1(,K9M,
–! %070M0&9,)&,0,+"0.?,
–! 0*IM#%I,(0*,(#*7")M'79,%070,7#,7?07,+"0.?,
•! *9$,9*119&=,"9&#/:9,&049,9*119&=,M')/%,&(?940&,e#*7#/#+)9&f,
–! %070,(0*,M9,%#*079%,07,0*I,149,e%"):9lMI,&#'"()*+f,
–! '&9,8bL,eÄ,W2-VbLf,7#,Z'9"I,7?9,%070,
•! V9J9"9*(9,
–! $$$AJ"99M0&9A(#4,,,
W'440"I5,6070,8#%9/,c,L0*+'0+9&,
•! 690/,$)7?,'*(9"70)*7Ig,
–! 9Q.#&9,7#,9*%,'&9",e0..,%9:9/#.9"f,#",*#7g,
•!
•!
•!
•!
•!
69(#5,*#,
!"#$%6N5,*#,
b'"C5,*#,
;/#+5,I9&,
B"99M0&95,I9&,Y,*#,
–! $?#,%9`*9&,"9&#/'1#*g,
•!
•!
•!
•!
•!
69(#5,&(?940,%9&)+*9",
!"#$%6N5,,&I&794,eJ#",*#$f,
b'"C5,H6B,%9&)+*9",
;/#+5,,&I&794,
B"99M0&95,&I&794=,("#$%,e('"07#"&f,
W'440"I5,6070,8#%9/,c,L0*+'0+9&,
•! K?07,0"9,7?9,J#'*%01#*&g,
–! 6070,8#%9/,0*%,b'9"I,L0*+'0+9,
•!
•!
•!
•!
•!
69(#5,,WbL=,"9/01#*0/,4#%9/,
!"#$%6N5,,WbL=,"9/01#*0/,4#%9/,
b'"C5,WbL=,"9/01#*0/,4#%9/,
;/#+5,,"9/01#*0/,4#%9/=,$#"C#$,e"'/9&f,
B"99M0&95,,]W940*1(,K9M^,e+"0.?,y,W2-VbL,%)0/9(7f,
•! K?9*,)&,%070,("#$%l&#'"(9%g,
•!
•!
•!
•!
•!
69(#5,,&)%9l9x9(7,#J,."#(9&&)*+,Z'9")9&,
!"#$%6N5,&)%9l9x9(7,#J,."#(9&&)*+,Z'9")9&,
b'"C5,&)%9l9x9(7,#J,."#(9&&)*+,Z'9")9&,Y,$#"C#$,
;/#+5,&(?9%'/9%,:&A,%"):9lMI,("#$%&#'"()*+,e!b2f,
B"99M0&95,,(#*1*'#'&,)*%9.9*%9*7,#J,Z'9"I,."#(9&&)*+,
W'440"I5,6070,8#%9/,c,L0*+'0+9&,
•! ;#$,7#,&.9()JI,?#$,7#,+97,%070g,
•! 69(#5,,J97(?,"'/9&=,2,497?#%&,MI,&(?940,%9&)+*9",
•! !"#$%6N5,,0**#701#*&,MI,&(?940,%9&)+*9"=,&94)l
0'7#401(0//I,+9*9"079%,794./079&,
•! b'"C5,,G-WE,/0*+'0+9,J#",H6B&,
•! ;/#+5,e6070/#+l/)C9f,"'/9&,
•! B"99M0&95,&I&794,e."9%`*9%,-2S&,7#,(#*7")M'79f,
•! R.9*,:&A,(/#&9%l$#"/%,0&&'4.1#*g,
•! 69(#5,#.9*,$#"/%,MI,J97(?)*+,]0*(?#"&^,
•! !"#$%6N5,#.9*,$#"/%,MI,!VRK6,70M/9&,
•! b'"C5,,#.9*,$#"/%,MI,H6B,
•! ;/#+5,#.9*,e/)C9,!"#$%6Nf,
•! B"99M0&95,,#.9*,e/)C9,W940*1(,K9Mf,
W'440"I5,6070,8#%9/,c,L0*+'0+9&,
•! H&9",:)9$,:&A,&I&794,:)9$,#*,%070g,
•! 69(#5,,(#*(9.7'0/,:&A,0(7'0/,&(?940,
•! !"#$%6N5,,4#&7/I,*#,%)x9"9*(9=,(0(?9,J#",(#4.0")&#*&,,
•! b'"C5,,4#&7/I,*#,%)x9"9*(9,
•! ;/#+5,,."#:9*0*(9,J#",?'40*,7"0*&0(1#*&,J#",0//#$)*+,
7#,("#$%l&#'"(9,0,:)9$,
•! B"99M0&95,,'*C*#$*,,e."#.")970"I,&I&794f,
b'9"I,2"#(9&&)*+,R:9":)9$,
•! H&9",S*79"J0(9,X9*9"01#*,
–! '&0M)/)7I,)&&'9&,eb'"Cf,
–! &#'"()*+,&#'"()*+,#Mi9(7&,0("#&&,70M/9&,e!"#$%6Nf,
•! R.14)[01#*&,
–! $?07,7#,#.14)[9,J#",
–! +04)*+,7?9,40"C97,
•! !"#$%l&#'"(9%,#.9"07#"&,
–! +"0.?,&90"(?,eW(##.f,
–! &#"1*+,
–! k,
HS,X9*9"01#*,
•! -/4#&7,&'..#"7,+9*9"01#*,#J,HS&,
–! b'"C5,,&.9()0/,794./079&,J#",%)x9"9*7,70&C&,
–! !"#$%6N5,,+9*9"079,HS&,J"#4,&(?940,
–! 69(#5,+9*9"079,HS&,J"#4,J97(?,"'/9&,,egf,
–! ;/#+5,,."0+40&,)*,("#$%l&#'"()*+,"'/9&,
–! B"99M0&95,%9%)(079%,HS&,J#",&.9()`(,70&C&,
•! -/4#&7,0//,&I&794&,0"9,&94)l0'7#401(,,
–! 0//#$,%9:9/#.9",7#,7$90C,."9l+9*9"079%,HS,
!"#$%6N,OQ04./9&,
!"#$%,G0M/9&,
!"#$%,!#/'4*&,
Please fill out the missing
company data!
IBM
Name
Headquarter
address
Please find a professor
and fill in her data
Name
Please fill out the missing
professor data
Carey
Name
E-Mail
E-Mail
Department
Department
Please fill out the missing
company data!
Name
IBM
$)7?,."9%)(079,,
.'&?l%#$*,
Please fill out the missing
professor data
Carey
Name
Headquarter
address
Size
Submit
Submit
Submit
Email
Large
Submit
$)7?,J#"9)+*,C9I&,
,
Please fill out the missing
professor data
Carey
Name
Email
Department
CS
Submit
$)7?,J#"9)+*,C9I&,
,
Department
CS
add
Please fill out the missing
department data
Name
Phone
Submit
Submit
%9*#"40/)[9%,
rr,
W#49,N9&7,2"0(1(9&,J#",HS&,
•! ;0:9,#'7.'7,`9/%&,`"&7=,7?9*,)*.'7,`9/%&,e!"#$%6Nf,
–! %#,*#7,4)Q,)*.'7Y#'7.'7,`9/%&,)*,0,&)*+/9,J#"4,
–! #'7.'7,`9/%&,."#:)%9,(#*79Q7,J#",7?9,9Q.9(79%,)*.'7,
•! ;0:9,0,]*#*9,#J,7?9,0M#:9^,M'u#*,
–! #7?9"$)&9=,$#"C9"&,9*79",+0"M0+9,$?9*,7?9I,0"9,/#&7,
•! K#"C9",J01+'9,J#",/#$,&9/9(1:)7I,."9%)(079&,eb'"Cf,
–! &I&794,0%%&,0"1`()0/,].#&)1:9&^,.9")#%)(0//I,
–! $#"C9",%#9&,*#7,+97,)*7#,7?9,"?I7?4,#J,]*#^,0//,7?9,149,
–! ?9/.&,$)7?,Z'0/)7I,(#*7"#/,
•! 2/0I,+049&5,,9A+A=,OW2,
–! p,*#7,0/$0I&,7?9,M9&7,)*(9*1:9,
•! &T;;+5+R8B5=+OB5;+LE+8B6B58FK+=L+OLbbb+
b'9"I,R.14)[01#*,
•! K?07,7#,#.14)[9,J#"g,
–! (#&7,epf,,
–! "9&.#*&9,149,
–! ."9()&)#*,e(9"70)*7I,7?07,0*&$9"&,0"9,(#""9(7f,
•! R7?9",]#*/)*9^,(#*&)%9"01#*&,
–! $?#/9,Z'9"I,"9&'/7,
–! &#49,"9&'/7&,$)7?,?)+?,(#*`%9*(9,
–! 0//,"9&'/7&,e*#,(#*`%9*(9,+'0"0*799&f,,
•! K?I,)&,7?)&,%)}('/7g,
–! ("#$%,+):9&,/)u/9,+'0"0*799&,0M#'7,149=,."9()&)#*,
–! *#,+##%,(#&7,4#%9/,9Q)&7&,J#",7?9,("#$%,
N07(?)*+,
•! S%905,K?)/9,I#',07,)7k,
–! ?#")[#*70/5,,0&C,J#",M)"7?,%079,0*%,./0(9,07,#*(9,
–! :9"1(0/,<5,,0&C,J#",M)"7?,%079,#J,&9:9"0/,."#J9&&#"&,
–! :9"1(0/,>5,,7?"99l$0I,(#4.0")&#*&,J#",&#"7,
•! X):9&,")&9,7#,."9J97(?)*+,0*%,*9$,0/+#")7?4&,
–! 49"+9,i#)*,~~,Z')(C&#"7g,
–! &#'"(9,)*J#,9:9*,7?#'+?,*#7,"9/9:0*7,J#",Z'9"I,
•! e-../)(0M/9,7#,0/4#&7,0//,&I&794&f,
N07(?)*+,
•! S%905,K?)/9,I#',07,)7k,
–! ?#")[#*70/5,,0&C,J#",M)"7?,%079,0*%,./0(9,07,#*(9,
–! :9"1(0/,<5,,0&C,J#",M)"7?,%079,#J,&9:9"0/,."#J9&&#"&,
–! :9"1(0/,>5,,7?"99l$0I,(#4.0")&#*&,J#",&#"7,
•! X):9&,")&9,7#,."9J97(?)*+,0*%,*9$,0/+#")7?4&,
–! 49"+9,i#)*,~~,Z')(C&#"7g,
–! &#'"(9,)*J#,9:9*,7?#'+?,*#7,"9/9:0*7,J#",Z'9"I,
•! e-../)(0M/9,7#,0/4#&7,0//,&I&794&f,
K?9"9,7#,)*:9&7g,
•! X97,7?9,*049,#J,7$#,X9"40*,(#4.#&9"&,
95@B+
N997?#:9*,
8#[0"7,
X9"&?$)*,
FL:9=8>+
!\HLL,
!\HLL,
HW-,
•! G$#,0/79"*01:9,&7"079+)9&5,$?)(?,)&,M9u9"g,
–! ("#$%l&#'"(9,7?9,(#'*7"I,#J,C*#$*,(#4.#&9"&,
–! ("#$%l&#'"(9,*9$,eX9"40*f,(#4.#&9"&,
X04)*+,7?9,2/03#"45,OQ04./9&,
•! W97,.")(9,0((#"%)*+,7#,](#4.911#*^,
–! 0,.9**I,4#"9,.9",70&C,4)+?7,*#7,?9/.,
–! $0)7,J#",7?9,")+?7,4#49*7,
–! ?)+?,"9$0"%&,0u"0(7,&.0449"&,
•! H*%9"&70*%,?#$,70&C&,0"9,.'M/)&?9%,c,J#'*%,
–! -8G,&90"(?,"0*C&,MI,q?)7&,.9",+"#'.,
–! 7?07,.0"04979",(0*,M9,(#*7"#//9%,MI,&I&794,
É,;SG&,!#4./979%,:&A,2")(9,
UA_,
pUAUF,,
pUAUD,,
pUAU>,,
pUAU<,,
UAw,
0B8FB9=5RB+LE+2(36+FL@A;B=BO+
UAm,
UAF,
UAD,
UA>,
UA<,
U,
U,
<U,
>U,
DU,
FU,
mU,
wU,
/<9:=B6+
zB"0*C/)*,97,0/=,WSX8R6,>U<<{,
nm,
ÉK)7?,~t,<,-&&+*7,!#4./979%,
0B8FB9=5RB+LE+2(36+=K5=+K5MB+5=+;B56=+L9B+566<R9@B9=+FL@A;B=BO+
<UUÉ,
nUÉ,
rUÉ,
_UÉ,
wUÉ,
pUAU<,,
mUÉ,
pUAU>,,
pUAUD,,
FUÉ,
pUAUF,,
DUÉ,
>UÉ,
<UÉ,
UÉ,
U,
<U,
>U,
DU,
FU,
mU,
wU,
3<@B+W@<96X+
zB"0*C/)*,97,0/=,WSX8R6,>U<<{,
nw,
V9&.#*&9,G)49,:&A,X"#'.,W)[9,
<>,
V9&.#*&9,G)49,B)"&7,;SG,
V9&.#*&9,G)49,B)"&7,<U,;SG&,
V9&.#*&9,G)49,B)"&7,>m,;SG&,
<U,
"B6AL96B+3<@B+W@<96X+
r,
w,
F,
>,
U,
<U,
>m,
mU,
<UU,
>UU,
FUU,
):@JB8+LE+2(36+<9+5+2(3+R8L:A+
zB"0*C/)*,97,0/=,WSX8R6,>U<<{,
n_,
;SG&,!#4./979%,.9",X"#'.,
<UUÉ,
FnAD,
>DAm,,
nUÉ,
2(36+FL@A;B=BO+W\X+
n_AD,
<wDAm,,
rUÉ,
_Ar,
_UÉ,
wUÉ,
>>DAr,
mUÉ,
FUÉ,
<U,
>m,
mU,
<UU,
>UU,
FUU,
):@JB8+LE+2(36+<9+5+2(3+R8L:A+
zB"0*C/)*,97,0/=,WSX8R6,>U<<{,
nr,
80*0+)*+,b'0/)7I,
G)4,E"0&C0,
nn,
b'9"I,R.14)[01#*,
•! K?07,7#,#.14)[9,J#"g,
–! (#&7,epf,,
–! "9&.#*&9,149,
–! Z'0/)7IY."9()&)#*,e(9"70)*7I,7?07,0*&$9"&,0"9,(#""9(7f,
•! R7?9",]#*/)*9^,(#*&)%9"01#*&,
–! $?#/9,Z'9"I,"9&'/7,
–! &#49,"9&'/7&,$)7?,?)+?,(#*`%9*(9,
–! 0//,"9&'/7&,e*#,(#*`%9*(9,+'0"0*799&f,,
•! K?I,)&,7?)&,%)}('/7g,
–! ("#$%,+):9&,/)u/9,+'0"0*799&,0M#'7,149=,."9()&)#*,
–! *#,+##%,(#&7,4#%9/,9Q)&7&,J#",7?9,("#$%,
b'9"I,R.14)[01#*,
•! K?07,7#,#.14)[9,J#"g,
–! (#&7,epf,,
–! "9&.#*&9,149,
–! P:5;<=>IA8BF<6<L9+WFB8=5<9=>+=K5=+596NB86+58B+FL88BF=X+
•! R7?9",]#*/)*9^,(#*&)%9"01#*&,
–! $?#/9,Z'9"I,"9&'/7,
–! &#49,"9&'/7&,$)7?,?)+?,(#*`%9*(9,
–! 0//,"9&'/7&,e*#,(#*`%9*(9,+'0"0*799&f,,
•! K?I,)&,7?)&,%)}('/7g,
–! ("#$%,+):9&,/)u/9,+'0"0*799&,0M#'7,149=,."9()&)#*,
–! *#,+##%,(#&7,4#%9/,9Q)&7&,J#",7?9,("#$%,
;#$,!0*,s#',G"'&7,7?9,!"#$%g,
<U>,
;#$,!0*,s#',G"'&7,s#'",O4./#I99e&fg,
<UD,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,
–! -.."#:0/,V079,Y,694#+"0.?)(,V9&7")(1#*&,
–! b'0/)`(01#*,G9&7,
–! X#/%,W97&Y;#*9I,2#7&,
–! V9%'*%0*(I,
–! a9")`(01#*YV9:)9$,
–! @'&1`(01#*Y-'7#401(,a9")`(01#*,
•! b'9"I,W.9()`(,G9(?*)Z'9&,
•! K#"C9",V9/01#*&?).,80*0+949*7,
<UF,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,,
–! .AA8LM5;+"5=B+I+%B@LR85AK<F+"B6=8<FTL96+
–! b'0/)`(01#*,G9&7,
–! X#/%,W97&Y;#*9I,2#7&,
–! V9%'*%0*(I,
–! a9")`(01#*YV9:)9$,
–! @'&1`(01#*Y-'7#401(,a9")`(01#*,
•! b'9"I,W.9()`(,G9(?*)Z'9&,
•! K#"C9",V9/01#*&?).,80*0+949*7,
<Um,
-.."#:0/,V079,c,
,694#+"0.?)(,V9&7")(1#*&,,
2"#&5,
•! O0&I,7#,&97'.,
•! G"0*&.0"9*7,
!#*&5,
•! O0&I,7#,%9J907,,
•! !0'&9&,0,/#7,#J,7"#'M/9,
<Uw,
&'()*+",-".//012(*23/"4')(05*6"
G'"C9",-}*)7I,0*%,O""#"&,
%#!"
%!!"
$#!"
$!!"
#!"
!"
!"
!!"
#!"
$!"
%!"
&!"
'!"
(!"
)!"
3:87B8+"597+
7'+8*+"
<U_,
zB"0*C/)*=,E#&&40**=,E"0&C0=,V049&?=,v)*5,!"#$%6N5,-*&$9")*+,b'9")9&,$)7?,!"#$%&#'"()*+A,#$%&'(=>U<<{,
&)!$
%!!"!!#$
-!"!!#$
&!!$
,!"!!#$
+!"!!#$
%)!$
*!"!!#$
)!"!!#$
%!!$
(!"!!#$
'!"!!#$
)!$
&!"!!#$
./01/2345/$621700/13$
89:;/0$7<$=>>?52:/23>$@9;:?A/B$
G'"C9",-}*)7I,0*%,O""#"&,
%!"!!#$
!$
!"
!!"
#!"
$!"
%!"
&!"
'!"
(!"
)!"
!"!!#$
3:87B8+"597+
C90D/0$
<Ur,
zB"0*C/)*=,E#&&40**=,E"0&C0=,V049&?=,v)*5,!"#$%6N5,-*&$9")*+,b'9")9&,$)7?,!"#$%&#'"()*+A,#$%&'(=>U<<{,
-.."#:0/,V079,,
,,,,,,,,,,,,,,,,,,,,,,,,
;SG,X"#'.,Ñ, ,S,"9(9*7/I,%)%,cdd+2(36+EL8+=K<6+8BP:B6=B8Ak,RJ,7?9,
>nn,;SG&,S,(#4./979%=,ee+LE+=KB@+NB8B+8BSBF=BO+$)7?#'7,0*I,
"90&#*,M9)*+,+):9*A,08<L8+=L+=K<6+(+L9;>+K5O+ef+8BSBFTL96^+5+gc\+
8BSBFTL9+85=Bg+(+F:88B9=;>+K5MB+hicc+6:J@<VBO+2(36^+N<=K+5+
jgk\+"9i9(1#*,"079,0o9",7?9,"9i9(1#*&,J"#4,7?)&,"9Z'9&79",e>m,7#70/,
"9i9(1#*&fA,S,?0:9,0u94.79%,7#,(#*70(7,7?9,"9Z'9&79",0*%,$)//,
'.%079,)J,S,"9(9):9,0,"9&.#*&9A,H*1/,7?9*,M9,:9"I,$0"I,#J,%#)*+,0*I,
$#"C,J#",7?)&,"9Z'9&79"=,0&,)7,0..90"&,7?07,7?9I,58B+8BSBFT9R+5JL:=+
e+<9+BMB8>+cl+2(36+JB<9R+6:J@<VBOg+,,,,.#&79%,MI,k,
J0)"5>,Y,m,,,,J0&75F,Y,m,,,,.0I5>,Y,m,,,,,(#445U,Y,m,
<Un,
zB"0*C/)*=,E#&&40**=,E"0&C0=,V049&?=,v)*5,!"#$%6N5,-*&$9")*+,b'9")9&,$)7?,!"#$%&#'"()*+A,#$%&'(=>U<<{,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,,
–! -.."#:0/,V079,Y,694#+"0.?)(,V9&7")(1#*&,
–! Z:5;<mF5TL9+3B6=+
–! X#/%,W97&Y;#*9I,2#7&,
–! V9%'*%0*(I,
–! a9")`(01#*YV9:)9$,
–! @'&1`(01#*Y-'7#401(,a9")`(01#*,
•! b'9"I,W.9()`(,G9(?*)Z'9&,
•! K#"C9",V9/01#*&?).,80*0+949*7,
<<U,
b'0/)`(01#*,
<<<,
b'0/)`(01#*&,e(7%f,
<<>,
b'0/)`(01#*&,e(7%f,
•! 2"#&,
–! a9"I,.#$9"J'/,
–! !0*,M9,'&9%,J#",7"0)*)*+,e(#*79Q7=,C*#$/9%+9=,kf,
•! !#*&,
–! 80I,7'"*,#x,$#"C9"&=,?'"7,(#4./91#*,149,
–! \#7,0/$0I&,0../)(0M/9,e9A+A=,#.)*)#*&Yi'%+)*+f,
–! X##%,Z'0/)`(01#*,(?#)(9,)&,?0"%,,
<<D,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,,
–! -.."#:0/,V079,Y,694#+"0.?)(,V9&7")(1#*&,
–! b'0/)`(01#*,G9&7,
–! *L;O+&B=6I2L9B>+0L=6+
–! V9%'*%0*(I,
–! a9")`(01#*YV9:)9$,
–! @'&1`(01#*Y-'7#401(,a9")`(01#*,
•! b'9"I,W.9()`(,G9(?*)Z'9&,
•! K#"C9",V9/01#*&?).,80*0+949*7,
<<F,
X#/%,&97&,Y,;#*9I,2#7&,
•! X#/%,%9"):9%,J"#4,
–! OQ.9"7&,
–! !"#$%,'&)*+,?)+?,Z'#"'4,
•! S*79"i9(7,7"0.,Z'9&1#*&,
•! N/#(C,'&9"&,)*,7"0.,0*%,)*:0/)%079,
0*&$9"&,
•! 2"#&,
–! Ro9*,:9"I,9x9(1:9,
–! !#&7,9}()9*7,
•! !#*&,
–! \#7,0/$0I&,0../)(0M/9,
–! 6)++)*+,+#/%,)&,?0"%,
<<m,
,
69J901*+,;#*9I,2#7&5,"9!-2G!;-,
<<w,
5,2"#+"04401(,X#/%,
Accuracygold
N missed
= 1!
N shown
W?#$,G0&C,
eX#/%,#",\#"40/f,
2"#:)%9,B99%M0(C,
J#",X#/%,,
K0"*,'&9",)J,
-(('"0(I,Ö,7$0"*,
<<_,
K#"C9",(0*,
(#4./0)*,0M#'7,
+#/%,0*&$9",
N/#(C,'&9",)J,
-(('"0(I,Ö,7"9i9(7,
zR/9&#*,97,0/5,2"#+"04401(,X#/%5,G0"+97,0*%,W(0/0M/9,b'0/)7I,-&&'"0*(9,)*,!"#$%&#'"()*+A,FG'&H=>U<<{,
5,X9*9"01*+,X#/%,
S%9*1JI,$#"C9",9""#"&,7#,40*'0/,0'%)7&,
69`*9,0,&97,#J,%070l4'701#*,J'*(1#*&,
!#//9(7,&97,#J,'*)7&,$)7?,C*#$*,(#""9(7,0*&$9"&,
H&9,+#/%,70"+97,%)&7")M'1#*,eL9,97,0/A,>U<Uf,7#,
("9079,4#"9,+#/%,
F,)79"01#*&,
<A!
>A!
DA!
FA!
<<r,
Gold
10 manual
60 programmatic
208 programmatic
233 programmatic
Units
213
3,004
3,103
15,509
Accuracy MAX work/worker
95.0%
100
89.0%
1,000
94.0%
1,500
92.5%
1,500
Gold Ratio
1:10
1:17
1:8
1:7
Time (hr)
4
5
9
27
Accuracy at
confidence 1.0
100.0%
96.0%
97.0%
97.0%
zL9,97,0/5,O*&'")*+,Z'0/)7I,)*,("#$%&#'"(9%,&90"(?,"9/9:0*(9,9:0/'01#*5,G?9,9x9(7&,#J,7"0)*)*+,Z'9&1#*,
%)&7")M'1#*A,!WO=,>U<U{,
zR/9&#*,97,0/5,2"#+"04401(,X#/%5,G0"+97,0*%,W(0/0M/9,b'0/)7I,-&&'"0*(9,)*,!"#$%&#'"()*+A,FG'&H=>U<<{,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,,
–! -.."#:0/,V079,Y,694#+"0.?)(,V9&7")(1#*&,
–! b'0/)`(01#*,G9&7,
–! X#/%,W97&Y;#*9I,2#7&,
–! "BO:9O59F>+
–! a9")`(01#*YV9:)9$,
–! @'&1`(01#*Y-'7#401(,a9")`(01#*,
•! b'9"I,W.9()`(,G9(?*)Z'9&,
•! K#"C9",V9/01#*&?).,80*0+949*7,
<<n,
V9%'*%0*(I5,b'#"'4,a#79&,
•! V9./)(079,7?9,70&C,e9A+A=,D,149&f,
•! H&9,40i#")7I,7#,%979"4)*9,7?9,")+?7,:0/'9,
•! 2"#&5,
–! O0&I,7#,)4./949*7,
–! ;0"%,7#,%9J907,
•! !#*&5,
–! \#7,0/$0I&,0../)(0M/9,e#.)*)#*&=,97(Af,
–! S*("90&9%,(#&7,
–! 40&C)*+,(0&9&,#J,04M)+')7I,#",%):9"&)7I=,]70)/^,
M9?0:)#"&,
–! 6#9&,*#7,(#:9",M)0&,
<>U,
z"9!-2G!;-=,!"#$%6N=,b'"C=,k,{,
!"#$%6N,j,W)4./9,b'#"'4&,
_+!58B>^+
++@SF58B>n<F6g:F<gBO:^+
++!L@A:=B8+&F<B9FB^++
++WdfdX+hcfClfcl+`+
MTJoin
(Dep)
p.dep = d.name
_++!58B>^+
+++@SF58B>n<F6g:F<gBO:^++
+++!L@A:=B8+&F<B9FB+`+
MTProbe
(Professor)
name=Carey
<><,
Please fill out the missing
Pleasedepartment
fill out the missing
data
Pleasedepartment
fill out the missing
data
Department
department data
CS
Department
Name
CS
Department
Name
Computer Science
Name
Phone
Phone
Phone
(949)
824-7427
Submit
Submit
Submit
Please fill out the missing
Please professor
fill out the missing
data
Please fill
out the missing
professor
data
Carey
Nameprofessor data
Carey
Name
Carey
Name
E-Mail
E-Mail
E-Mail
Department
[email protected]
Department
Dep
Computer
Submit Science
Submit
Submit
zenFnf,r>Fl_F>_{=,
znFn,l,r>,F_,F>,_{=,
z,nFn,l,<>,DF,mw,_{,
z@SF58B>n<F6g:F<gBO:=,!L@A:=B8+&F<B9FB{=,
z4i(0"9IÜ)(&A'()A9%'=,!#4.'79",W()9*(9{=,
z@SF58B>n<F6g:F<gBO:=,O/9(7")(0/,O*+)*99")*+{=,
z4i(0"9IÜ)(&A'()A9%'=,O/9(7")(0/,O*+)*99")*+{=,
z(0"9IÜM90A(#4=,!L@A:=B8+&F<B9FB{,
z(0"9IÜM90A(#4=,!#4.'79",W()9*(9{,
zB"0*C/)*=,E#&&40**=,E"0&C0=,V049&?=,v)*5,!"#$%6N5,-*&$9")*+,b'9")9&,$)7?,!"#$%&#'"()*+A,#$%&'(=>U<<{,
6)&0+"9949*7,
•! OQ04./9&5,
–! -//,%)x9"9*7,0*&$9"&,
–! D,#'7,#J,m,0+"99,#*,0*&$9",-=,7?9,#7?9",>,#*,
0*&$9",N,
•! !#'*79"lG9(?*)Z'95,
–! H&9,9Q.9"7&,7#,M"90C,19&,
–! 6I*04)(0//I,(#//9(7,4#"9,i'%+49*7&,
<>>,
S4."#:)*+,b'#"'4&,j,L0M9/)*+,70&C&,
S*79"l"079","9/)0M)/)7I,890&'"9&,
•! !#*&9*&'&,9&14079&,
,
,
,
,
,
,
,
,,
<>D,
–! 2'".#&95,OQ0(7,0+"9949*7,
–! W701&1(&5,29"(9*7,0+"9949*7=,!#?9*á&,C0..0=,W(#uà&,2)=,k,
zW794/9",97,0/5,2"0(1(0/,-&&9&49*7A,I676/=;9"J"K</5C/E412">UUF{,
29"(9*7,0+"9949*7,
<UU,*049&=,>,"079"&5,
L0M9/&5,]I9&^,l~,)&,0,(#4.0*I,
,,,,,,,,,,,,,]*#^l~,)&,*#7,0,(#4.0*I,,
N,
-,
0B8FB9=5RB+.R8BB@B9=+
2"e0fteFUyDUfY<UU,t,UA_U,
,
<>F,
s9&,
\#,
s9&,
FU,
>U,
\#,
<U,
DU,
!#?9*á&,C0..0,
<UU,*049&=,>,"079"&5,
L0M9/&5,]I9&^,l~,)&,0,(#4.0*I,
,,,,,,,,,,,,,]*#^l~,)&,*#7,0,(#4.0*I,,
N,
-,
29"(9*70+9,-+"9949*7,2"e0f,
2"e0fteFUyDUfY<UU,t,UA_U,
,
,
V079",-,&0I&,]s9&^,wUÉ,#J,7?9,149,
V079",N,&0I&,]s9&^,mUÉ,#J,7?9,149,
,
2"e9f,t,,2"#M0M)/)7I,#J,"0*%#4,0+"9949*7,
,,,,,,,,,,t,eUAwâUAmf,y,eUAFâUAmf,t,UAm,
,
E,,,,,,,t,,eUA_,j,UAmf,Y,e,<,j,,UAmf,t,UAF,
<>m,
s9&,
\#,
s9&,
FU,
>U,
\#,
<U,
DU,
S4."#:)*+,b'#"'4&,j,L0M9/)*+,70&C&,
S*79"l"079","9/)0M)/)7I,890&'"9&,
•! !#*&9*&'&,9&14079&,
–! 2'".#&95,OQ0(7,0+"9949*7,
–! W701&1(&5,29"(9*7,0+"9949*7=,!#?9*á&,C0..0=,W(#uà&,2)=,k,
•! !#*&)&79*(I,9&14079&,
–! !#*&)&79*7/I,0../I,0,&(#")*+,"'M")(,
–! W701&1(&5,290"&#*h&,"=,W.90"40*h&,"?#=,kA,
•! 890&'"949*7,9&14079&,,
,
<>w,
–! X#0/,)&,7?9,."9&9":9,0&,4'(?,)*J#"401#*,0&,.#&&)M/9,J"#4,90(?,
i'%+9,0*%,7#,)*(#".#"079,7?07,)*J#"401#*,)*7#,7?9,4#%9/,
–! W701&1(&5,2")*().0/,(#4.#*9*7&,0*0/I&)&=,X9*9"0/)[0M)/)7I,7?9#"I,
zW794/9",97,0/5,2"0(1(0/,-&&9&49*7A,I676/=;9"J"K</5C/E412">UUF{,
2"#M0/)&1(,X"0.?)(0/,8#%9/,
-**#707#",
*#)&9,
2"9&9*(9Y-M&9*(9,
#J,0*,#Mi9(7,)*,7?9,
β
)40+9,
zi
images
!"#$
a9(7#",#J,
θz e?)%%9*f,
0u")M'79&,
α
σj
wj
S*%):)%'0/,
(#4.#*9*7,
$9)+?7,
τj
S*%):)%'0/,
7?"9&?#/%,
M
yij
xi
annotators
γ
lij
L0M9/,J#",
)40+9,S,J"#4,
0**#707#",i,
ij
Ji
N
labels
|Lij |
-,:9"&)#*,#J,7?9,&)+*0/,
(#""'.79%,MI,
0**#707#",0*%,)40+9,
&.9()`(,*#)&9,
<>_,
zK9/)*%9",2,97,0/5,G?9,8'/1%)49*&)#*0/,K)&%#4,#J,!"#$%&A,L$H#=>U<U{,
2"#M0/)&1(,X"0.?)(0/,8#%9/,
G0C&5,W9/9(7,)40+9&,(#*70)*)*+,07,/90&7,#*9,%'(C,
x2i
1
_mAFÉ,
2
3
S%9*1`9%,D,+"#'.&,#J,0**#707#"&,
•! X"#'.,<5,%)x9"9*1079&,%'(C&,J"#4,9:9"I7?)*+,9/&9,
•! X"#'.,>5,%'(C&,0*%,+"9M9&,J"#4,9:9"I7?)*+,9/&9,
•! X"#'.,D5,%'(C&=,+"9M9&=,0*%,+99&9,J"#4,9:9"I7?)*+,9/&9,
<>r,
x1i
_mAFÉ,(#""9(7,
80i#")7I,:#795,,
wrADÉ,(#""9(7,
zK9/)*%9",2,97,0/5G?9,8'/1%)49*&)#*0/,K)&%#4,#J,!"#$%&A,L$H#=>U<U{,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,,
–! -.."#:0/,V079,Y,694#+"0.?)(,V9&7")(1#*&,
–! b'0/)`(01#*,G9&7,
–! X#/%,W97&Y;#*9I,2#7&,
–! V9%'*%0*(I,
–! aB8<mF5TL9I"BM<BN+
–! @'&1`(01#*Y-'7#401(,a9")`(01#*,
•! b'9"I,W.9()`(,G9(?*)Z'9&,
•! K#"C9",V9/01#*&?).,80*0+949*7,
<>n,
a9")`(01#*,
Please find a department
University
UC Berkeley
Name
Phone
Submit
•! 2"#&,
–! a9"I,9x9(1:9,
–! -../)(0M/9,J#",0,:0")97I,#J,'&9,(0&9&,
–! ;0"%,7#,(?907,
•! !#&7,
<DU,
–! OQ.9*&):9,e149,y,(#&7f,
–! N)0&,,
Is this a valid department?
Is this a valid department?
University
UC Berkeley
Is this a valid department?
University
UC Berkeley
Name
EECS
University
UCeecs.berkeley.edu
Berkeley
Name
EECS
URL
Name
EECS
URLPhone
eecs.berkeley.edu
(510)
642-3214
URLPhone Noeecs.berkeley.edu
(510) 642-3214
Yes
Phone No
(510) 642-3214
Yes
No
Yes
W#I/9*75,8'/1lL9:9/,V9:)9$,
<D<,
zN9"*&79)*,97,0/5,W#I/9*75,-,K#"%,2"#(9&&#",$)7?,0,!"#$%,S*&)%9A,M$#N2,>U<U{,
W#I/9*75,B)*%lB)Qla9")JI,
•! 2"#+"044)*+,("#$%&,7#%0I,)&,?0.?0[0"%=,
&)4)/0",7#,HS,79(?*#/#+I,M9J#"9,%9&)+*,
.0u9"*&,
/)C9,8#%9/la)9$l!#*7"#//9",
•! B)*%lB)Qla9")JI,)&,0,%9&)+*,.0u9"*,,
J#",."#+"044)*+,("#$%&,7#,(#4./979,,
#.9*l9*%9%,70&C&,
<D>,
zN9"*&79)*,97,0/5,W#I/9*75,-,K#"%,2"#(9&&#",$)7?,0,!"#$%,S*&)%9A,M$#N2,>U<U{,
W#I/9*75,B)*%lB)Qla9")JI,
Find
“Identify at least one area
that can be shortened without
changing the meaning of the
paragraph.”
S*%9.9*%9*7,0+"9949*7,7#,)%9*1JI,.07(?9&,
Fix
“Edit the highlighted section to
shorten its length without
changing the meaning
of the paragraph.”
Soylent, a prototype...
V0*%#4)[9,#"%9",#J,&'++9&1#*&,
Verify
<DD,
“Choose at least one rewrite
that has style errors, and
at least one rewrite that
changes the meaning
of the sentence.”
zN9"*&79)*,97,0/5,W#I/9*75,-,K#"%,2"#(9&&#",$)7?,0,!"#$%,S*&)%9A,M$#N2,>U<U{,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,,
–! -.."#:0/,V079,Y,694#+"0.?)(,V9&7")(1#*&,
–! b'0/)`(01#*,G9&7,
–! X#/%,W97&Y;#*9I,2#7&,
–! V9%'*%0*(I,
–! a9")`(01#*YV9:)9$,
–! o:6TmF5TL9I.:=L@5TF+aB8<mF5TL9+
•! b'9"I,W.9()`(,G9(?*)Z'9&,
•! K#"C9",V9/01#*&?).,80*0+949*7,
<DF,
R7?9",
•! @'&1`(01#*YB99%M0(CY!#449*7&,
–! ;9/.&,7#,%979(7,04M)+')19&Y'*(/0")7I,
–! !0*,M9,'&9%,MI,V9:)9$,70&C&,7#,%979"4)*9,)J,7?9,
$#"C9","90//I,7")9%,
–! \RGO5,SJ,I#',%#*h7,.'7,0,,
(#449*7,M#Q=,7?9I,$)//,,
$")79,7?94,&#49$?9"9|,
•! -'7#401(,!?9(C&,
–! -../)(0M/9,)*,(0&9&,$?9"9,,
`*%)*+,0,&#/'1#*,)&,?0"%,,
M'7,:9")JI)*+,)&,90&I,
–! OQ04./95,2"#79)*,B#/%)*+,
e?u.5YYJ#/%A)7Yf,,
<Dm,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,,
–! -.."#:0/,V079,Y,694#+"0.?)(,V9&7")(1#*&,
–! b'0/)`(01#*,G9&7,
–! X#/%,W97&Y;#*9I,2#7&,
–! V9%'*%0*(I,
–! a9")`(01#*YV9:)9$,
–! @'&1`(01#*Y-'7#401(,a9")`(01#*,
•! Z:B8>+&ABF<mF+3BFK9<P:B6+
•! K#"C9",V9/01#*&?).,80*0+949*7,
<Dw,
!#&7,:&A,b'0/)7I,
<D_,
z80&#*,97,0/5,B)*0*()0/,S*(9*1:9&,0*%,7?9,à29"J#"40*(9,#J,!"#$%&h=,;!R82=,>UUn{,
b'0/)7I,0&,0*,R.14)[01#*,X#0/,
V01*+,W#"7,
!#4.0")&#*,W#"7,
(%
!"#$
!"'$%
!"'%
!"&$%
!"&%
!"#$%
%
!%
,-./01%
<Dr,
)!%
20.3/4.54%
*!%
%$&'!($
67./089$:%
67./089+:%
+!%
201;-<4%
&!%
,-=4%
z80"('&,97,0/5,;'40*l.#$9"9%,W#"7&,0*%,@#)*&=,aL6N=,>U<>{,
H&9",S*79"J0(9,:&A,b'0/)7I,
!
!
Professor
Please fill out the missing
professor data
MTJoin
(Professor)
p.name = "carey"
MTProbe(Dep)
Department CS
name
E-Mail
Submit
Please fill out the missing
department data
Department
Name
Department
MTJoin
(Dep)
p.dep = d.name
MTProbe
(Professor)
name=Carey
Department
Name
CS
Please fill out the missing
professor data
Phone
Submit
Please fill out the missing
professor data
Carey
Name
E-Mail
MTProbe
(Professor, Dep)
name=Carey
Carey
Name
E-Mail
Department
Department
Phone
Submit
Department
Phone
Submit
e69.0"749*7,`"&7f,
<Dn,
p.dep=d.name
Please fill out the missing
department data
Carey
N ame
name="Carey"
Submit
e2"#J9&&#",`"&7f,
pej\+,88L8C"5=B,
e69l*#"40/)[9%,2"#M9f,
phj\+,88L8C"5=B,
zB"0*C/)*=,E#&&40**=,E"0&C0=,V049&?=,v)*5,!"#$%6N5,-*&$9")*+,b'9")9&,$)7?,!"#$%&#'"()*+A,#$%&'(=>U<<{,
R'7/)*9,
•! X9*9"0/,G9(?*)Z'9&,,
–! -.."#:0/,V079,Y,694#+"0.?)(,V9&7")(1#*&,
–! b'0/)`(01#*,G9&7,
–! X#/%,W97&Y;#*9I,2#7&,
–! V9%'*%0*(I,
–! a9")`(01#*YV9:)9$,
–! @'&1`(01#*Y-'7#401(,a9")`(01#*,
•! b'9"I,W.9()`(,G9(?*)Z'9&,
•! $L87B8+"B;5TL96K<A+/595RB@B9=+
<FU,
20I)*+,N-6,G'"C9"&g,
•! 20I,J#",M0%,$#"Cg,
–! 2"#&5,,
•! L9&&,?0&&/9,$)7?,7'"C9",40*0+949*7,
•! M9u9","9Z'9&79","9.'701#*,
•! Ro9*,*#7,&'"9=,$?#,)&,$"#*+,e"9Z'9&79"Y7'"C9"f,
–! !#*&5,,
•! V9.'701#*,J#",.0I)*+,9:9"I7?)*+,
•! '*%9"4)*9&,"9.'701#*,&I&794,
•! H&9,M#*'&,0&,)*(9*1:9,)*&790%,#J,"9i9(1#*&,
•! W.0449"&,l~,N/#(C,J"#4,J'7'"9,70&C&,
<F<,
K#"C9",V9/01#*&?).,80*0+949*7,
38:6K+C+,9=B8+=KB+95@B+LE+L9B+LE+=KB+ij+'&+6=5=B6+
K?07à&,7?07,0//,0M#'7g,G?9"9,0"9,4#"9,7?0*,>UU,;)7&,0:0)/0M/9=,,
6L+<E+(+N8<=B+Q;L8<O5+cel+T@B6+(+RB=+A5<O+EL8+=K5=g,K9)"%A,
llllllllllllll,
G?9Ià"9,0//,+#*9,*#$g+(+N56+=B@A=BO+=L+=8L;;+MI,.'Å*+,%#$*,&7'x,/)C9,ä2'9"7#,V)(#ä=,äX'04ä=,
0*%,äa)"+)*,S&/0*%&^,
llllllllllllll,
G"'&?,?0&,.'7,7?9&9,'.,)*,7?9,.0&7A,Sà:9,0*&$9"9%,J#'",#",`:9,149&,M9J#"9=,M'7,Sà:9,0/$0I&,+):9*,
%)x9"9*7,&7079&,0*%,K5MB9q=+6:J@<VBO+@L8B+=K59+i+<9+5+J5=FKggg(+OL9q=+N59=+=L+6:J@<=+5+
K:9O8BO+59O+K5MB+=KB@+5;;+8BSBF=BOg+S,?0:9,*#,(/'9,$?07,G"'&?à&,"9i9(1#*,.#/)(I,)&,J#",&'M4)Å*+,
<UU,;SG&AAA0//,#J,7?9,G'"C#.1(#*,"9:)9$&,70/C,0M#'7,&$)o,.0I=,M'7,%#*à7,49*1#*,?#$,40*I,#J,7?9,
;SG&,7?9I,%)%=,&#,Sà4,&1//,$0"I,#J,%#)*+,7##,40*IA,
llllllllllllll,
S,$#'/%,$#*%9",$?I,0,"9Z'9&79",)&,.'Å*+,#'7,;SG&,/)C9,7?9&9A,.8B+=KB>+=8><9R+=L+FL;;BF=+M5;<O+
NL87B8+(%6=,J#",9Q04./9g,SJ,&#49#*9,(0*,7?)*C,#J,0,+##%,"90&#*,J#",0,;SG,/)C9,7?)&,Sà%,M9,+/0%,7#,
?90",)7|,,,
llllllllllllll,
2"9&)%9*7,RM040,&0I&,7?9"9,0"9,m_,&7079&A,;#$,$)//,7?07,$#"Cg,
<F>,
,
K#"C9",V9/01#*&?).,80*0+949*7,e(7%f,
K0)7=,$?07g,m_,&7079&g,6)%,G9Q0&,"9l%):)%9,)*7#,79"")7#")9&,0+0)*,#",&#497?)*+g,
llllllllllllll,
S7à&,0,"9J9"9*(9,7#,0,+0x9,N0"0(C,RM040,40%9,$?9*,?9,$0&,0,M)7,7##,1"9%,J"#4,7?9,(04.0)+*,
7"0)/A,S7à&,J'**I,?#$,.9#./9,C99.,"9J9"")*+,7#,7?07,)*&790%,#J,?)&,4#"9,0+"0*7,4)&70C9&=,/)C9,
(#*1*')*+,7?9,4)&0%:9*7'"9&,)*,S"0Z,0*%,-J+?0*)&70*=,7?"#$)*+,4#*9I,07,K0//,W7"997,)*&790%,#J,
)*:9&1*+,)7,)*,)*J"0&7"'(7'"9=,0*%,7"901*+,V9.'M/)(0*&,$)7?,"9&.9(7A,
llllllllllllll,
N0""Ià&,79/9."#4.79",7#/%,?)4,7?9"9,$9"9,m_,&7079&A,N0""I,&0I&,$?079:9",7?9,79/9."#4.79",79//&,
?)4,7#,&0I,M9(0'&9,$?9*,N0""I,$#*%9"&,#x,&(").7,."#M/94&,&##*,J#//#$A,G?9"9,0"9,#7?9",J'**I,
+0x9&,#'7,7?9"9A,s#'7'M9,?0&,0,M'*%/9,#J,7?94A,
llllllllllllll,
W7079,#J,!#*J'&)#*=,W7079,#J,8)*%=,W7079,#J,69\)/9AAAA,
llllllllllllll,
H?l#?A,2#/)1(&,?0&,9*79",7?9,7?"90%A,@'&7,$07(?,7?9,%)&('&&)#*,%)&)*79+"079,)*7#,*049,(0//)*+A,
llllllllllllll,
s90?=,S,+'9&&,?9,ä4)&'*%9"9&14079%ä,7?9,%)}('/7I,#J,&.90C)*+,)*,.'M/)(=,'*/)C9,7?07,?'+9,
)*79//9(7,$?#,%9(/0"9%,ä8)&&)#*,-((#4./)&?9%ä,$0I,M0(C,$?9*AAA,
,<FD,
,
K#"C9",V9/01#*&?).,80*0+949*7,e(7%f,
!?9(C,#'7,N0""I,"9J9"")*+,7#,0,+'I,)*,7?9,4)/)70"I,0&,0,ä(#".&940*äA,V9.9079%/IA,K0I,7#,+#=,M)+,
+'IA,s#',$#*%9",)J,?9,.)(C9%,'.,7?#&9,:#(0M'/0"I,&C)//&,)*,S*%#*9&)0,0&,0,C)%,#",)J,)7,i'&7,(#49&,
*07'"0//I,7#,?)&,40&&):9,ä)*79//9(7äA,s#',C*#$=,7?07,)*79//9(7,7?07,."9:9*79%,?)4,J"#4,$")1*+,
0*I7?)*+,0&,."9&)%9*7,#J,7?9,/0$,"9:)9$,07,;0:0?%A,
!/0"C5,\0?=,Sà4,*#7,+#)*+,7#,(0//,0*I#*9,?9"9,*049&A,S,/)C9,9:9"I#*9,?9"9,0*%,"9&.9(7,7?9,?9/.,
Sà:9,"9(9):9%,J"#4,7?9,.#&79"&A,Sà//,7#&&,0,J9$,/)+?79*)*+,M#/7&=,7?#'+?A,
llllllllllllll,
RC0I=,M0(C,7#,7?9,"90&#*,J#",7?9&9,;SG&,l,&#49,#J,I#',+0:9,&#49,.#&&)M/9,0*&$9"&,&#,S,+'9&&,
7?9"9,(#'/%,M9,0,/9+)14079,'&9,J#",7?9,%070A,,,W#49,#J,7?9&9,;SG&,"90//I,40C9,49,$#*%9"=,
7?#'+?AAAA,
llllllllllllll,
-&,04'&)*+,0&,)7,4)+?7,M9,7#,&#49,.9#./9,7?07,ä(#".&9ä,0*%,ä(#".&ä,0"9,?#4#.?#*9&=,?9,$0&,
"9J9"")*+,7#,?)4,0&,0,(#".&,40*,$?)(?,)&,79(?*)(0//I,0(('"079A,K?)/9,)7,)&,0$C$0"%,)7,)&*à7,/0'+?,
#'7,/#'%,J'**I,(#4.0"9%,7#,7?9,+94&,$9,?90"%,J"#4,N'&?,0*%,J"#4,&#49,#J,7?9,.9#./9,
(04.0)+*)*+,J#",."9&)%9*7,")+?7,*#$A,
llllllllllllll,
@'&7,JI)=,(#"9,0*%,(#".&,0"9,?#4#.?#*9&A,G?9,ä(#".&,#J,(0%97&ä,)&,."#*#'*(9%,ä(#"9,#J,(0%97&Aä,
G?9,ä."9&&,(#".&ä,)&,ä."9&&,(#"9äA,K?9*,'&9%,0&,0,./'"0/=,I#'à%,&0I,7?9,&,#*,7?9,9*%=,M'7,'*/)C9,
<FF,
kkkkk,
R7?9",."0(1(0/,1.&,
•! ]O07,I#'",#$*,%#+lJ##%^,
•! 8#*)7#",%)&('&&)#*,J#"'4&,
–! G'"C9",\01#*5,?u.5YY7'"C9"&A."#M#0"%&A(#4,
–! ?u.5YY$$$A7'"C0/9"7A(#4,q,
–! G'"C#.1(#*5,"9.#"7Y0:#)%,&?0%I,"9Z'9&7#"&,
–! -40[#*,B#"'4,J#",87'"C,
•! -%%"9&&,J99%M0(C,
•! 80C9,7?9,."#(9&&,0&,7"0*&.0"9*7,0&,.#&&)M/9,
e.0I49*7=,:0/)%01#*=,)*79*7=,kf,
<Fm,
Crowdsourced Data Management
Applications
!!
Relational
–! information extraction
–! schema matching
–! entity resolution
–! data spaces
–! building structured KBs
–! sorting
–! top-k
–! ...
!!
Beyond relational
–! graph search
–! classification
–! mobile image search
–! social media analysis
–! question answering
–! NLP
–! text summarization
–! sentiment analysis
–! semantic wikis
–! ...
146
Human-Assisted Graph Search /
Classification
Human-Assisted Graph Search: It’s Okay to Ask
Questions, A. Parameswaran, A. Das Sarma, H. GarciaMolina, N. Polyzotis, J. Widom.VLDB-11
147
Human Assisted Graph Search
!!
Given a DAG G
!! Containing unknown target nodes
!! Find target nodes by asking humans
search questions at nodes in G
"! “Is there a target node reachable from the current node?”
!!
Applicable to many problems
!! classification, workflow debugging, filter synthesis,
interactive search
148
Example: Classifying Images
into a Taxonomy
To classify image of an accord car
Is the image a type
of vehicle?
vehicle
car
nissan
maxima
honda
YES!
Is the image a
type of toyota?
NO!
toyota
sentra
Is the image a
type of honda?
YES!
target node = intended category
Is the image a type of X? = Is the target node reachable from X?
149
Classification Problem Setting
!! Don’t
want to ask all nodes or serially
!! too expensive or too slow
!! Given
a limit of k questions, find
the best k nodes to ask in parallel
!! Assuming humans answer correctly, processing k
answers yields a superset of the target nodes
!! we want to minimize the size of this superset
!! Practical
usage
!! to classify into a huge taxonomy, first ask k questions
!! is the superset small enough that some in-house
humans can do a final processing?
!! If not, ask next k questions, and so on
150
Solution
!! Given
a limit of k questions, find
the best k nodes to ask in parallel
!! Reduces
to a partition problem
!! given an undirected tree, find k edges
such that their deletion minimizes the
size of the largest connected component
!! Runs
in O(nlogn) time (n = # of nodes)
!! Paper
considers many other practical settings
!! multiple target nodes, unlimited resources, etc.
151
Mobile Image Search
CrowdSearch: Exploiting Crowds for Accurate and RealTime Image Search on Mobile Phones. T. Yan, V.
Kumar, D. Ganesan. MobiSys-2010
152
Mobile Image Search
Automated
image search
engine
153
CrowdSearch: Use Crowdsourcing
Yes, Yes, No, No, No
Automated
image search
engine
No, Yes, Yes, Yes, Yes
Yes, No, Yes, Yes, No
154
Money vs. Delay Tradeoffs
!!
To minimize money
–! send tasks sequentially
–! increases delay
!!
Yes, Yes, No, No, No
To minimize delay
–! send tasks in parallel
–! increases money
!!
Goal: return one
validated image before
deadline, while
minimizing money
No, Yes, Yes, Yes, Yes
Yes, No, Yes, Yes, No
155
Key Idea
!!
Send out the first task
!!
Suppose has received No, Yes, No
Need to receive Yes, Yes in order to have a match
Compute P(Yes, Yes | No, Yes, No)
Compute P(receive the above seq before deadline)
If their product < threshold !
first task likely to fail,
send out the second task
!!
!!
!!
!!
!!
And so on ...
156
Probability Estimation
!!
P(Yes, Yes | No, Yes, No)
P(receive the above seq before deadline)
!!
Model the crowd behavior
!!
–! what is the probability that a user will answer Yes, Yes, given No, Yes,
No?
–! given a task, how long would it take until a user starts answering?
–! given an answer on a task, how long until the next answer arrives?
!!
!!
Use training data to estimate model parameters
Use the model to estimate the above probabilities
157
Event Detection and Monitoring in
Social Media
Work done and deployed at Kosmix, now @WalmartLabs
158
Event Dection
Detector 1
Twitter
Foursquare
Detector 2
…
Detector n
Candidate
events
Candidate
events
Candidate
events
Event
evaluator
and
ranker
Ranked
events
•! Sample heuristics
–! exploit keyword popularity & correlations
•! Egypt/revolt, Egypt/protest, Egypt/Tahrir
–! exploit foursquare checkins
–! exploit Twitter users who often tweet upcoming events
•! cnnbrk
Event Dection: Add Crowdsourcing
Detector 1
Twitter
Foursquare
Detector 2
…
Detector n
Candidate
events
Candidate
events
Candidate
events
Event
evaluator
and
ranker
Ranked
events
•! For each event
–! find users who tweet about the event
–! if do not find enough users, drop event
•! Can be viewed as “implicit crowdsourcing”
Event Dection: Add Crowdsourcing
Detector 1
Twitter
Foursquare
Detector 2
…
Detector n
Candidate
events
Candidate
events
Candidate
events
Population 1
Event
evaluator
and
ranker
Ranked
events
Population 2
Population 3
...
•! For each event, show
–! phrase pairs: Egypt/revolt, Egypt/protest, Arab/spring
–! sample tweets, most popular URLs
•! Ask: is this an interesting event?
•! Different populations have different quality
–! show “high-quality” events first on Tweetbeat
Event Monitoring: Current Solutions
Baltimore shooting
@dsmith: Baltimore shooting on TV5!
Egyptian uprising
@far213: Tahrir is packed!
•! Manually write rules to match tweets to events
–! e.g., tweet contains certain keywords / userids " positive
–! conceptually simple, relatively easy to implement
–! often achieve high initial precision
•! Limitations
–! expensive, don’t scale
–! manually writing good rules can be hard
–! rules often become invalid/inadequate over time
•! e.g., Baltimore shooting " John Hopkins shooting
164
Event Monitoring: Kosmix Solution
Event
Twitter firehose
Baltimore
shooting
Initial profile
{Baltimore, shoot}
Learning algorithm
New profile
{Baltimore, shoot, John Hopkins}
165
Tweets
“Baltimore shooting on TV5!”
“Baltimore shooting. John Hopkins
shut down.”
...
Major Problem 1:
Constructing Good Initial Profile
•! Event as output by event detector:
–! phrase pairs: Egypt/revolt, Egypt/protest, Arab/spring
–! sample tweets, most popular URLs
•! Can automatically infer initial profile from these
•! But accuracy is limited " difficult to learn from
•! Solution: use crowdsourcing
–! ask “is this an interesting event?”
–! if yes, ask “list people/organization/location involved”
–! use answers to craft initial profile
–! turned out the best profile should include people/org/loc
166
Second Major Problem:
Preventing Overgeneralization
•! Also known as snowballing
–! start with “Baltimore shooting”
–! soon learn profile subsuming all current events in
Baltimore
•! Well-known serious problem in machine learning
•! Solution: use crowdsourcing
–! build multiple levels of generalization into learning algo
–! humans monitor tweet streams of an event
–! if detect snowballing, alert algorithm
–! algorithm automatically backtracks, and moves to a more
conservative level of generalization
167
Summary: The Critical Roles of
Crowdsourcing in Three Places
verify
event detector
events
create
initial profile
monitor
snowballing
learning algorithm
168
V9&90"(?,!?0//9*+9&,
G)4,E"0&C0,
<wn,
X"#'*%,N"90C)*+,!?0*+9&,
$)7?,%)x9"9*7,)*79*7&Y4#1:01#*,
<_U,
X"#'*%,N"90C)*+,!?0*+9&,e(7%f,
!/#&9%lK#"/%,
<_<,
R.9*lK#"/%,
X"#'*%,N"90C)*+,!?0*+9&,e(7%f,
<_>,
X"#'*%,N"90C)*+,!?0*+9&,e(7%f,
#include <iostream.h>!
!
int main(){!
int num1;!
cout << "Welcome to Tom's Odd and
!
Even Evaluator Game!!\n\n";!
cout << "Enter an integer\n\n”;!
cin >> num1;!
if ( num1 % 2 == 0 )!
cout << "\nGuess what?\n\n" << num1
<< " is an even number!\n\n"; !
if ( num1 % 2 != 0 )!
out << "\nGuess what?\n\n" << num1
<< " is an odd number!\n\n”; !
return 0;!
!
}!
<_D,
!
!
X"#'*%,N"90C)*+,!?0*+9&,e(7%f,
<_F,
V9&90"(?,-+9*%0,
•! 6070,8#%9/Yb'9"I,
L0*+'0+9&,
•! b'9"I,R.14)[01#*,
•! b'9"I,OQ9('1#*,
•! b'0/)7I,!#*7"#/,
•! W7#"0+9Y!0(?)*+,
•! H&9",)*79"J0(9&,
•! \9$,&#"7Yi#)*,0/+#")7?4&,
-/+#")7?4&,
<_m,
•! G'"C9",N9?0:)#",
•! K#"C9",V9/01#*&?).,
80*0+949*7,
•! S*79"0(1:)7I,
•! 2/03#"4&,
•! ;IM")%,80(?)*9,L90"*)*+Y
;'40*,-/+#")7?4,
•! V9.9070M)/)7I,
•! kAA,
V9.9070M/9,OQ.9")49*7&,
200 HITs,
(!"
'!"
!"#$%&'()*%
&!"
)!*!#""
%!"
)!*!$""
)!*!%""
)!*!&""
$!"
)!*!'""
#!"
!"
#"
<_w,
$"
%"
&"
+%,-./01%(.012"34%
'"
6N,#,!"#$%&#'"()*+yy,
!8LNOC258O+08LJ;B@6Y+
•!
08LR85@@<9R+159R:5RBY+*'(+
b'9&1#*,%9&)+*=,04M)+')19&=,+"0*'/0")7I=,k,
•!
WbLg,K?I,
*#7g,
/59>^+@59>+79LJ6+=L+=:89+
2")(9=,"9./)(01#*,J0(7#"=,;SG,+"#'.,&)[9=,9Q.)"01#*,149=kA,
•!
!K59R<9R+A;5rL8@+JBK5M<L8+
S*("90&)*+,40"C97,&)[9=,*9$,./03#"4,J907'"9&=k,
•!
o:9R;B+LE+O<sB8B9=+=BFK9<P:B6+
V9i9(1#*,.#/)(I,eb'#"'4la#79=,G9&7lW97=kf=,,b'0/)7I,(#*7"#/,eb'#"'4=,)79"01:9,4#%9/&=kf,
•!
•!
1B589<9R+BsBF=6+I+!L@@:9<=>+/595RB@B9=+
k,
3KB+%HC.AA8L5FK+
•!
%5=5+<9OBAB9OB9FB++
•!
%H/&+LAT@<UB6+P:B8<B6++
SJ,;K,(?0*+9&=,0..,*99%,*#7,(?0*+9,
–! 69()%9,$?07,7#,("#$%&#'"(9,
–! W701&1(&,0M#'7,7?9,40"C97,./0(9=,,
Z'9&1#*,#"%9")*+=k,
<__,
V9&90"(?,-+9*%0,
•! 6070,8#%9/Yb'9"I,
L0*+'0+9&,
•! b'9"I,R.14)[01#*,
•! b'9"I,OQ9('1#*,
•! b'0/)7I,!#*7"#/,
•! W7#"0+9Y!0(?)*+,
•! H&9",)*79"J0(9&,
•! \9$,&#"7Yi#)*,0/+#")7?4&,
-/+#")7?4&,
<_r,
•! G'"C9",N9?0:)#",
•! K#"C9",V9/01#*&?).,
80*0+949*7,
•! S*79"0(1:)7I,
•! 2/03#"4&,
•! ;IM")%,80(?)*9,L90"*)*+Y
;'40*,-/+#")7?4,
•! V9.9070M)/)7I,
•! kAA,
V9J9"9*(9&,
#=KB8+3:=L8<5;6+
•! 2A,S.9)"#1&=,80*0+)*+,!"#$%&#'"(9%,;'40*,!#4.'701#*=,KKK,T<<=,80"(?,>U<<A,
•! RA,-/#*&#=,8A,L90&9=,!"#$%&#'"()*+,J#",S*J#"401#*,V97")9:0/5,2")*()./9&=,897?#%&=,0*%,-../)(01#*&=,WSXSV,
@'/I,>U<<A,
/LTM5T9R+,?5@A;B6+
•!
•!
•!
•!
•!
\0&0,!/)(C$#"C9"&=,?u.5YY(/)(C$#"C9"&A0"(A*0&0A+#:Y7#.,
B#/%)7=,?u.5YYJ#/%A)7Y,
X0/0QI,P##=,?u.5YY$$$A+0/0QI[##A#"+Y,
W70(C#:9"#$=,?u.5YY&70(C#:9"#$A(#4Y,
B"99M0&9=,?u.5YY$$$AJ"99M0&9A(#4Y,
35?L9L@>++
•! -A,@A,b')**,0*%,NA,NA,N9%9"&#*5,];'40*,!#4.'701#*5,-,W'":9I,0*%,G0Q#*#4I,#J,0,X"#$)*+,B)9/%^=,!;S,
>U<<A,
•! -*?0),6#0*=,V0+?',V040C")&?*0*=,-/#*,sA,;0/9:I=,!"#$%&#'"()*+,WI&794&,#*,7?9,K#"/%lK)%9,K9MA,!-!8,
>U<<,
•! $$$A("#$%&#'"()*+"9&'/7&A(#4,
0;5rL8@6+
•!
•!
•!
•!
•!
-40[#*,89(?0*)(0/,G'"C=,?u.&5YY$$$A47'"CA(#4Y,
OW2,X-8O=,?u.5YY$$$A+$0.A(#4Y+$0.Y+049&2"9:)9$Y9&.+049Y,
20)%,!"#$%&#'"()*+=,?u.5YY$$$A&40"7&?997A(#4Y,
W040W#'"(9=,?u.5YY&040&#'"(9A#"+Y,
!"#$%B/#$9"=,?u.5YY("#$%#$9"A(#4Y,
<_n,
V9J9"9*(9&,(7%A,
!8LNO6L:8F<9R+%H/&6+
•! !"#$%6N,
–! -4M9",B9*+=,8)(?09/,B"0*C/)*=,6#*0/%,E#&&40**=,G)4,E"0&C0=,W04,80%%9*=,W'C")1,V049&?=,-*%"9$,
K0*+=,0*%,V9I*#/%,v)*5,!"#$%6N5,b'9"I,2"#(9&&)*+,$)7?,7?9,aL6N,!"#$%A,aL6N,>U<<,
–! 8)(?09/,B"0*C/)*=,6#*0/%,E#&&40**=,G)4,E"0&C0=,W'C")1,V049&?=,0*%,V9I*#/%,v)*5,!"#$%6N5,
-*&$9")*+,b'9")9&,$)7?,!"#$%&#'"()*+A,WSX8R6,>U<<,
•! 69(#,
–! K)%#4,97,0/A,!"#$%!"#$%,>U<<=,?u.&5YY&)79&A+##+/9A(#4Y&)79Y7?9("#$%("#$%Y,
•! b'"C5,
–! 80"('&=,OA,K'=,6A,E0"+9"=,WA,80%%9*=,0*%,VA,!A,8)//9"5,;'40*l.#$9"9%,W#"7&,0*%,@#)*&A,2aL6N,>U<>,
–! -A,80"('&=,OA,K'=,6A,E0"+9"=,WA,80%%9*=,0*%,VA,!A,8)//9"5,!"#$%&#'"(9%,6070M0&9&5,b'9"I,2"#(9&&)*+,
$)7?,29#./9A,!S6V,>U<<,
•! ;/#+,
–! KA,W?9*=,-A,6#0*=,@A,BA,\0'+?7#*=,VA,V040C")&?*0*5,69(/0"01:9,S*J#"401#*,OQ7"0(1#*,H&)*+,6070/#+,
$)7?,O4M9%%9%,OQ7"0(1#*A,aL6N,>UU_,
–! 2"9%)(079&AvA,!?0)=,NA,,a'#*+=,-A,6#0*=,@A,BA,\0'+?7#*5,O}()9*7/I,)*(#".#"01*+,'&9",J99%M0(C,)*7#,
)*J#"401#*,9Q7"0(1#*,0*%,)*79+"01#*,."#+"04&A,WSX8R6,>UUn,
•! B'&)#*!R82,
–! -A,8#")&?)40=,\A,W?)*0+0$0=,WA,8#(?)['C)5,G?9,2#$9",#J,S*79+"079%,-M&7"0(1#*,J#",6070l(9*7")(,
;'40*Y80(?)*9,!#4.'701#*&A,aL6W,>U<<,
<rU,
V9J9"9*(9&,(7%A,
"B;5TL95;+.AA;<F5TL96+
•! ;IM")%,6070,S*79+"01#*5,
VA,8(!0**=,KA,W?9*=,-A,6#0*5,807(?)*+,W(?940&,)*,R*/)*9,!#44'*)19&5,-,K9M,>AU,-.."#0(?A,S!6O,>UUr,
•! S*J#"401#*,OQ7"0(1#*5,
VA,;#x40*=,WA,-49"&?)=,@A,B#+0"7I=,EA,2079/=,6A,K9/%=,BA,K'5,-4./)JI)*+,!#44'*)7I,!#*79*7,!"901#*,$)7?,
8)Q9%lS*)101:9,S*J#"401#*,OQ7"0(1#*A,!;Sl>UUn,
•! 6A,69'7(?=,RA,X"99*&?.0*=,NA,E#&79*C#=,GA,8)/#5,H&)*+,80"C#:,!?0)*,8#*79,!0"/#,7#,./0I,G"):)0A,S!6O,>U<<,
HB>L9O+"B;5TL95;+
,
•! -A,20"049&$0"0*=,-A,60&,W0"40=,;A,X0"()0l8#/)*0=,\A,2#/I[#1&=,@A,K)%#45,FCB/1O-770786A"%=/39"#6/=;9P"
$8Q7"'R/D"84"-7R"SC67E4172"aL6N,>U<<,
•! GA,s0*=,aA,E'40"=,6A,X0*9&0*P"G=4:A#6/=;9P"KT3540E1."G=4:A7"U4="-;;C=/86"/1A"I6/5ON0B6"$B/.6"#6/=;9"
41"&4V056"H94167A,8#M)WI&l>U<U,
•! ;IM")%,S40+9,W90"(?5,
GA,s0*=,aA,E'40"=,6A,X0*9&0*5,!"#$%W90"(?5,OQ./#)1*+,!"#$%&,J#",-(('"079,V90/l149,S40+9,W90"(?,#*,
8#M)/9,2?#*9&A,8#M)&I&,>U<UA,,
<r<,
V9J9"9*(9&,(7%A,
Z:5;<=>+
•! 20*#&,S.9)"#1&5,N9,0,G#.,89(?0*)(0/,G'"C,K#"C9"5,s#',\99%,pm,0*%,m,8)*'79&=,?u.5YY$$$AM9?)*%l7?9l
9*94Il/)*9&A(#4Y>U<UY<UYM9l7#.l49(?0*)(0/l7'"Cl$#"C9"lI#'l*99%A?74/,>U<U,,
•! 6A,R/9&#*=,aA,;9&79"=,-A,W#"#C)*=,XA,L0'+?/)*=,@A,L9=,LA,N)9$0/%5,2"#+"04401(,+#/%5,70"+979%,0*%,&(0/0M/9,
Z'0/)7I,0&&'"0*(9,)*,("#$%&#'"()*+A,;!R82,>U<<,,
•! @#?*,L9=,-A,O%4#*%&=,aA,;9&79"<=,0*%,LA,N)9$0/%5,O*&'")*+,Z'0/)7I,)*,("#$%&#'"(9%,&90"(?,"9/9:0*(9,
9:0/'01#*5,G?9,9x9(7&,#J,7"0)*)*+,Z'9&1#*,%)&7")M'1#*A,!WO,>U<U,
•! WA,W794/9"5,-,(#4.0")&#*,#J,(#*&9*&'&=,(#*&)&79*(I=,0*%,490&'"949*7,0.."#0(?9&,7#,9&1401*+,
)*79""079","9/)0M)/)7IA,2"0(1(0/,-&&9&&49*7=,V9&90"(?,c,O:0/'01#*,>UUF,
•! 2A,K9/)*%9"=,WA,N"0*&#*=,WA,N9/#*+)9=,2A,29"#*05,G?9,8'/1%)49*&)#*0/,K)&%#4,#J,!"#$%&A,L$H#,>U<U,
•! 8A,N9"*&79)*=,XA,L)u/9=,,VA!A,8)//9"=,NA,;0"740**=,8A,-(C9"40*=,6A,VA,E0"+9"=,6A,!"#$9//=,EA,20*#:)(?5,
W#I/9*75,-,K#"%,2"#(9&&#",$)7?,0,!"#$%,S*&)%9A,HSWG,>U<U,
•! KA,80&#*,=,,6A,@A,K0u&5,B)*0*()0/,S*(9*1:9&,0*%,7?9,à29"J#"40*(9,#J,!"#$%&h=,;!R82,>UUn,
<r>,