!
"
#$%
&%
'%((

)''*
++,

-	$%(
(.&
/
	%	
&
&(%(



	

		
	

		
	
	
	


	

	

4	
51.	
0
	1
8	$(
:(


6%(
$

7#		
;	#$

8(1(
)$$#$(

<.(1	
)
.	$
7#		
'%(	

6%(
$

7#		
;	#$

=
$
"(&	
$.($



7#		
'%(	





	














	







	








	








	














	


	
























	







	











 



 

















	

	

!









	






	

"#$"%






"$&



'








!



























(



	













!












	


	



"#$"%




	
			
		


	

	

	

4	
51.	


8	$(
:(


6%(
	
%
7#	
8(1	
	#$($	
)$$#$
	
.(1

.	$
7#		

6%(
	
%
7#	
>11((
	
&
7#		





&
)

 


 
	


) 



 

)	
 
*

 


	 
*



)
	

*

*






+	

 
	 

) 

, 
*



	
*

 	

,



*

	


 	
	

* 	
 	
	
	
&


	
 



*




*
 


	
	
 


   
)
 

-


)	




 )


!
 


 
*

*
*



 

"#$"%
)

*




*



 

./01!




*

'



&



	

 


,

*

 


	



 

)
	*
 
"

 
*


  





?@(
	
$(
A	
1(%(&(1

.
$(
	#$
B
$	
%(
	$
$(
8($1
	1
'$	$$
(
((




(11%

CC+3


0*	%

	11
(2
	$3
.	3
4	
51.	

6%(
	
%
7#	
8(1	
	#$($	
)$$#$
	
.(1


.	$
5
	%
$


')
7#		
'%(	
(.	2
	3%1.	
9.3#3



43
51.	
	1
83
:(


* 
*




	

)


*

2


	




. 
	



"#$"%



##
3
4451



	

"	



	



	

	







.//.

	




	















	




















%






	






	
	


























	





	






6






7) 
.//8
	







	
	

.














	










	






	

#&

.


#&

$	1	
1


"





















	
	












*






6



'	

./99










	
	



:









	






	


















	

	








	































	















	














:










	


	

	













	








	





;





<

	










	






6



	





		







;










	
















	 


		


	
4=>


>=5

























<














?


	










	


		







	
















@

A


$





	


'	
;
./99



	










	
	


	

















	
















	













































	

	
	





















		
;




















*
.

.	

#&
D
.#$
#&

1	(
		








	

$








6


















	

	










	




	
	










	




	






6







	




(
	







	


	










<





!
















	

	














	






(
;






.//.




















	









































 





	








	








B



!
"#$"%









 




















 	














 















	










;

	


	

















	







=








	
=





















	


	

!










#	













!
























	









	




























	

















 


=
	




!"	
	

	

	

:

	








	










-

C
'
./95

'

C
#




./99












!







	

	
.



 





	











D


'	;
. 
E

;
.



!

	










E



7
!
'	


./>/





E








. 









F.G






(	;
.4


!







F4G



./>0






	







E


	

 






!








43
51.	
	1
83
:(


E
3
F.!.
. !
H
.G=.
.


.4
3
E.
!.
1
3
!.
4


E

;



	









. 

. 
F.G

#
EI3IE.



./84
+



	



.;









	

#
+I3I. !


	



J

./04








.;
#
JI3I+. !








	













!




K














!

















 













 !




@
#I3I. !



$
	


	

50


























	















	




/


5


L










	

'




	






	





	


	



















	


















.1







	







.4
	

6







$






'












.4
















%

		
	
	






.4




	








	








	



























	











!F1$
2
'







;
















	
















.4
	
@<

;





A


.
	
@<

;

	











A

!F1$
2
'







;
































	

?

.4



*
.

.	

#&
D
.#$
#&

1	(
		


	














	






.

	























'

		



	

(

	
			










.













	











	
			

			


















4































 













	


















E

;
E







	











>




















*













	
*
&










.4




	



111.


















@	(
3
<(	%#

#(
$#1(13














 

!



"
#


!

$!







%



$&

'
!$
 

!

"



(

$ 



$

$
)!*
 +

$
 







!

!
&
#!
$

,

-
!
,
*

 


$



	

#


&.

#






# 
#

*
!


&
%





$ 
&&

!
!
) +




//










!
!




@	(
3
"(
&$%(
$	$$


#(

	
.



	


. 


. 






	






	





. 
	


	
	





. 


. 










. 


. 
	













	


	


. 


	

	





. 


. 















43
51.	
	1
83
:(


@	(
3
G(1	H
E
%	#(


.H

	3




. 


	
















. 
















. 
















.



4

&311.
(




&





.1!81









	
















































*"

		
	

	

	




	
!







!

. 








'
C
(

./0>















-












7






	

















 

*

.


4






 

<

. 


. 






	
	




 






.


4







 








	
















3
.
'
@
3

;

. 
>














IM






















	
+'
'

'
'



M
3
F

!
+
G
=


5


+

3

.

4

H.4
=
5
8




3

.

4
!.
4
H.4
=
.55
L



	

	



IM




'
C
(


./0>
%

















			

			




F9G

F/G

















	























	


. 
. F!.
.GN

. 
3
.




	


	








	




3
!.






	













	






. 
3
F

!
+
G
=
F	

!
+
G
0



*
.

.	

#&
D
.#$
#&

1	(
		


	

3

.

4

H.
H4
=
L
9


	

3

.

4

H.
4H.
=
L
/















!





	



O





	









. 
;








. 









(	
.4




7


./08



7



C
'
./08




4!


K
C
'
./08

7



C
'
./08


"#$"%
	
!



#

	







	









"$&







	





5
<




	















IM



"#$"%









(	;
.4!























	




=
































 





	







!







	




























"	









. 
'
C
(

./0>










I>
3'
L
'
@



>!



	
!


















"#$"%
7



C
'
./08

7


C
'
./0/







@

A


@	(
3
>(A	
=*=F>*"=
$	(

I
	#$

$
	$
(%(
=<FJ
(1

	$
(%(
IJ
	1	&$(1

.
<(($$
K
'#	
C, 3


Source of Variation Sum of Products Test Statistic Interpretation 
Within Group Ax sAx ' sAx .2 
Ax 
Within Group Ay sAy' sAy .2 
Ay 
Within Group Bx sBx ' sBx .2 
Bx Agreement Within 
Within Group By sBy' sBy .2 
By Each Group 
Within Group Cx sCx ' sCx .2 
Cx 
Within Group Cy sCy' sCy .2 
Cy 

Group A vs. Group B 
Group A vs. Group C 
Group B vs. Group C 
(sAx + sAy)' (sBx + sBy) 
(sAx + sAy)' (sCx + SCy) 
(sBx + sBy)' (sCx + sCy) 

 AB 

 AC 

 BC 
Factor 1 Effect 
Group x vs. Group y 
Group Ax vs. Group By 
(sAx + sBx + sCx)' (sAy + sBy + 
sCy) 
sAx ' sBy 

 xy 

 AxBy 
Factor 2 Effect 
Group Ax vs. Group Cy sAx ' sCy 
 AxCy 
Group Bx vs. Group Ay sBx ' sAy 
 BxAy Factor 1 × Factor 2 
Group Bx vs. Group Cy sBx ' sCy 
 BxCy Interactions 
Group Cx vs. Group Ay sCx ' sAy 
 CxAy 
Group Cx vs. Group By sCx ' sBy 
 CxBy 
Total (sAx +...+ sCy)' (sAx +...+ sCy) .2 
Class Overall Agreement 


43
51.	
	1
83
:(





:





M








	

(

	


























>!
!!>!	

	










!





7


C
'
./0/

































	









	





	


	



	










	










	
 	
















	



















"$&
























	



. 













	



!







M


+

3
.
+

M
.1




3
F.

. 

=
*G
H
+
. 

..



	

 






	





. 
;




	


	




















	






7
6















!


6







IM


P

























. 




	



	
	

. 








!.

	













/











	



.


4


/
	

4


>


/
	

























	









'
















	


	







	












"#$"%





	





		





8













	















>



	

E;


	

























!

	






	
	

















@

A






"



















	













		




















@


A




	

E318>


!











<












	
















"#$"%























	

	!	

"$&




! 




*
.

.	

#&
D
.#$
#&

1	(
		

@	(
3
'#..	


$AA	
=*=F>*"=

(#$3


Source of Variation 
Group 1 vs. Group 2 
Sum of Products or E(
) and 
var(
) 
(s11 + s12)' (s21 + s22) 
Test 
Statistic Interpretation 
Agreement Within and 
Between Groups 1 and 2 
… (34 comparisons) … all 
 
* > 50 … 
Group 9 vs. Group 8 (s91 + s92)' (s81 + s82) 
Agreement Within and 
Between Groups 9 and 8 

All Groups 1 … 9 
E(
) = 1096134 
var(
) = 4019158 

 
* = 79.2 Agreement Within and 
Between Groups 
Condition 1 vs. Condition 2 
Group 1 Condition 1 vs. 
Group 2 Condition 2 
(s11 +...+ s91)' (s12 +...+ s92) 
s11 ' s22 

 
* = 667.6 Agreement Within and 
Between Conditions 
No Interaction Between 
Group 1 Cond. 1 and Group 2 
Cond. 2 
… (70 comparisons) … all 
 
* > 5 … 
Group 9 Condition 1 vs. 
Group 8 Condition 2 
All Interactions 
s91 ' s82 
E(
) = 1111133 
var(
) = 13069835 

 
* = 38.4 
No Interaction Between 
Group 9 Cond. 1 and Group 8 
Cond. 2 
No Interactions 
Total (s11 +...+ s92)' (s11 +...+ s92) .2 = 541.8 Overall agreement 









! 




'6



	




	


	







	






















	



	

















!




(










	










	
















310L5
&31115N
.31004
&3111>
"












	




	











	
	








	



	






















(

	
	

	
			
	
	

Q







!







	




	









	

6








	







 
:





.4!	











E









	
















!





	
















 





:
./01










	


E
.
E. 
.
"

-


'
./95





6!




	


#
J



!







	























43
51.	
	1
83
:(

j
@	(
3
=&&
I.	$(
1((
$(
%	

$A
(($(1

1	(
(($
.

 !"#$%$&%'&(
. 

. 
0
)	


+

1
*
2
	



3$

)	


+


1

*
2
		




.!	



	





	

 !


:













































	
















L
<
















	

	







!
	

















&!


E








.1!81







.1!8





	

/8R

























E








.

F1$













.
.
	#1($




/


	




	



	


	












	







!

	

<












	


	



























	








	










 







B
!























	












	











"#$"%



















!!
!
	


"
#$"%

















K









	















!











	






!





"#$"%











	







#
J


































	

	

	






<


?


	




!





"#$"%



	








		



!






	 








$











	






#
J
-


'

./95



	












	
	






*
.

.	

#&
D
.#$
#&

1	(
		
































:






	

















	

	











'
;

./8L








	 
















	!	




6




	

	





S6









	

	
	














@	



A
	
	





6



	


	




	


#;
./91


B;
./95










"$&





?

C
#
./95
(








	









	





















	



<












	














	













	

	

	






#

.//L



	

	
!





:





6





	










=












	














T








	










!






















	






	




	











$







	







6

	






	




	



TU
4114
4115










	








	

























	









(


4111
:



	

















































		








@A

@
A




















0
"
1	

:





%
B

%
(V''





<




#		

?


'
Q


"

	

	



	
!











6




%
-

K


+




	

	


7

T


B
	



'




-













	











	
:




%

W
')
Q


K 
 
%
	


-

%
	
&	
T2

'

<



K 
 
%
	


"

-

%
-	2
O

T2
'


<



K 
 
%
	





-




	







		

(







	










43
51.	
	1
83
:(


'
	

7


T
C
'
:V
./08
"#$"%









 
<


((1

'	
'$	$$
'($

$(
=.(
	
'$	$$	

=	$

>..!>.>


7


T
C
'
:V
./0/
#
	

*


 

N#
	

O1#	$	
'$	$$

4
.48!.>0


7) 
T
.//8
@(.(
	(	
%&%		
P;#1	.($	

	
#((Q3

K 
 
Q*

K 
 
(
*





#

"
.//L
>
1	
.($1

(	%
	
1	$	
		
B
"T
K

+
	


#
:T
./91


	$	
&	
	.($

$	$$
4

"
P
:



#
W
'


C
'
:+
.//0











	







B

#



'$
#$#
	
OR#	$
81(


5
4L.!494


(

B
4111
5#	
	$(
	
1	$	
#
''
<





?
'#
C
#
:T
./95

(








!





!


	


N#
	

$(
=.(
	
'$	$$	
=	$
,C
>90
LL9

L0>

TU
EW
4114

4115
'$
#$#
	
(R#	$
.1(
A$

1	
%	
	(

#
7)'LO7
F:::
	
G
QVK


	=

=


K
K
C
'
:V
./08
'	
















<.($
	

4
5.0!54>


B
V
./95
'$	$$
#

	
$
7

7





-

%
C
'
:V
./95
$
	






	




.($
	
C
.
.>>!.5.


'
:V
C
(

:?
./0>












.($
	
+
4
45/!489


'

'
./8L
*&	
	.($

$	$$
"
P
BW!?




'

'
C
#



T
./99
*&	
	.($

$	$$

$(
(	%
	
((
4


"
P
BW!?




'	

-
7
./99
"	






	


3
<
W
B
7


?
(

C
V

W
	
+
"
'	



4/.!>19
K

7

-



'




T
#
.//.
'	
)#((3
7	
$
Q

-

:
T
./01




*

	









'	;


E

;


<.($
	
,
>
L./!L40




&(L((%(12
33

'&
(($=(&$(12
,3C3