
    	i9*              	       $   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZmZm	Z	m
Z
mZ defdZdede
e   fdZdede
e   fdZd	e	e   defd
ZdddZdeeef   deee	eeef      f   fdZdeeef   deeef   fdZddZedk(  r e        yy)    N)datetimetimezone)AnyDictListOptionalTuplereturnc                      t        j                  t        j                        j	                  d      j                         S )Nr   microsecond)r   nowr   utcreplace	isoformat     E/home/openclawd/.openclaw/workspace/data_pipeline/raw_to_canonical.py_now_isor   
   s,    <<%--!-<FFHHr   vc                 F    | y t        |       j                         }|r|S d S N)strstrip)r   ss     r   	_norm_strr      s&    yAA1tr   c                 b   t        |       }|sy 	 |j                  dd      }t        j                  |      }|j                   |j                  t
        j                        }|j                  t
        j                        j                  d      j                         S # t        $ r |cY S w xY w)NZz+00:00)tzinfor   r   )
r   r   r   fromisoformatr   r   r   
astimezoner   	Exception)r   r   s2dts       r   _norm_tsr%      s    !AYYsH%##B'998<<0B}}X\\*22q2AKKMM s   BB   B.-B.partsc                     dj                  d | D              }t        j                  |j                  d            j	                         d d S )N|c              3   L   K   | ]  }t        j                  |d d        yw)TF)	sort_keysensure_asciiN)jsondumps).0ps     r   	<genexpr>z_stable_hash.<locals>.<genexpr>%   s      U!DJJqDuEEUs   "$utf-8   )joinhashlibsha1encode	hexdigest)r&   blobs     r   _stable_hashr9   $   s>    88UuUUD<<G,-779#2>>r   )defaultc                      |D ]  }||c S  | S r   r   )r:   
candidatescs      r   _objr>   )   s"     =H Nr   rawc                    t        | j                  dg             }t        | j                  dg             }t        | j                  dg             }t        | j                  dg             }| j                  dg       D ]  }t        |j                  d      |j                  d      |j                  d            }|j                  dg       xs g D ]:  }t        j                  |      }|j                  d|       |j                  |       <  |D ]  }t        |j                  d      |j                  d      |j                  d            }	t        |j                  d      |j                  d	            }
|j                  dg       xs g D ]N  }t        j                  |      }|j                  d|	       |
|j                  d|
       |j                  |       P  |D ]  }t        |j                  d      |j                  d
      |j                  d            }|j                  d      }|j                  d      }|j                  dg       xs g D ]b  }t        j                  |      }|j                  d
|       ||j                  d|       ||j                  d|       |j                  |       d  ||||dS )Nsessionstopicsmessageseventsid
session_idexternal_idtopic_idparent_session_id
message_idrA   rB   rC   rD   )listgetr>   copydeepcopy
setdefaultappend)r?   rA   rB   rC   rD   r   sidttctidtsidmmcmidmsidmtideecs                     r   _flattenr^   0   sp   CGGJ+,H#''(B'(FCGGJ+,H#''(B'(F WWZ$ 155;l 3QUU=5IJx$* 	Aq!BMM,,MM"	   155;j 11553GHAEE,'/B)CDz2&," 	 Aq!BMM*c*lD1OOB	    155;l 3QUU=5IJuu\"uuZ x$* 	Aq!BMM,,lD1j$/MM"		 	 r   c                 T  +, t        |       }g }g }g }g }i }i }i }i +i ,i }	i }
|j                         D ci c]  \  }}|t        |       }}}ddddd}|d   D ]  }t        |j	                  d            }t        t        |j	                  d      |j	                  d                  }t        t        |j	                  d      |j	                  d      |j	                  d	      |j	                  d
                  }t        t        |j	                  d      |j	                  d      |j	                  d                  }t        ||g      }|rd| gng }|j                  d|        t        +fd|D        d       }|r|dxx   dz  cc<   |}n:dt        |      dz   d}|j                  ||||||j	                  di       d       |D ]  }|+|<   	 |j	                  d      |j	                  d      |j	                  d      fD ]  }t        |      }|s|||<     |d   D ]  }t        |j	                  d            }t        t        |j	                  d      |j	                  d                  }|r|j	                  |      nd }t        t        |j	                  d      |j	                  d      |j	                  d                  }t        t        |j	                  d
      |j	                  d      |j	                  d                  }t        ||g      }|rd| gng }|j                  d|        t        ,fd|D        d       }|r|dxx   dz  cc<   |}n:dt        |      dz   d}|j                  ||||||j	                  di       d       |D ]  }|,|<   	 |j	                  d      |j	                  d       |j	                  d      fD ]  }t        |      }|s|||<     |d!   D ]  }t        |j	                  d            }t        |j	                  d            }t        |j	                  d             }|r|j	                  |      nd }|r|j	                  |      nd } t        t        |j	                  d"      |j	                  d#      |j	                  d$                  }!t        t        |j	                  d%      |j	                  d&      |j	                  d'                  }"t        t        |j	                  d      |j	                  d
      |j	                  d(                  }#|xr d| xs t        || |!|#|"g      }$|$|	v r|d!xx   dz  cc<   |	|$   }nAd)t        |      dz   d*}||	|$<   |j                  |||| |!|#|"|j	                  di       d+       |j	                  d      |j	                  d,      |j	                  d      fD ]  }t        |      }|s|||<     |d-   D ]   }%t        |%j	                  d            }t        |%j	                  d            }t        |%j	                  d             }t        t        |%j	                  d,      |%j	                  d.                  }&|r|j	                  |      nd }|r|j	                  |      nd } |&r|j	                  |&      nd }'t        t        |%j	                  d/      |%j	                  d0      |%j	                  d                  }(t        t        |%j	                  d      |%j	                  d
      |%j	                  d(                  }#t        |%j	                  d1      t              r|%j	                  d1      n(|%j                         D ci c]  \  }}|d2vs|| c}}})|xr d| xs t        |(|'|#|)g      }$|$|
v r|d-xx   dz  cc<   d3t        |      dz   d*}||
|$<   |j                  |||| |'|(|#|)d4        t        | j	                  d5            xs d6t               |t        |      t        |      t        |      t        |      d|d7t        |d8 9      t        |d: 9      t        |d; 9      t        |d< 9      d=}*|*S c c}}w c c}}w )>Nr   rK   rA   rG   titlenamestart_ts
start_time
started_at
created_atend_tsend_timeended_atzext:zhash:c              3   2   K   | ]  }|v s|     y wr   r   )r.   kseen_sessions     r   r0   znormalize.<locals>.<genexpr>   s     OQ,=NLOO   	
   ses_04dattrs)rE   rG   r`   rb   rf   rp   rE   rF   rB   rI   subject
created_tstsc              3   2   K   | ]  }|v s|     y wr   r   )r.   rj   
seen_topics     r   r0   znormalize.<locals>.<genexpr>   s     Ka1
?JqMKrl   top_)rE   rG   rF   r`   rr   rp   rH   rC   	author_iduser_id	sender_idtextbodycontent	timestampmsg_06d)rE   rG   rF   rH   rw   rs   rz   rp   rJ   rD   message_ref
event_typetypepayload>   rE   rs   ra   r   r   event_idrH   r}   re   r   rJ   rF   rG   r   evt_)rE   rG   rF   rH   rJ   r   rs   r   sourceunknown)r   generated_at
counts_rawcounts_canonicaldedup_droppedc                     | d   S NrE   r   xs    r   <lambda>znormalize.<locals>.<lambda>  
    qw r   )keyc                     | d   S r   r   r   s    r   r   znormalize.<locals>.<lambda>  
    1T7 r   c                     | d   S r   r   r   s    r   r   znormalize.<locals>.<lambda>  r   r   c                     | d   S r   r   r   s    r   r   znormalize.<locals>.<lambda>  r   r   )metarA   rB   rC   rD   )r^   itemslenr   rM   r>   r%   r9   rQ   next
isinstancedictr   sorted)-r?   flatout_sessions
out_topicsout_messages
out_eventssession_ref_maptopic_ref_mapmessage_ref_mapseen_message
seen_eventrj   r   
raw_countsr   r   extr`   rb   rf   hash_keykeysexisting_idcanon_idrefrrS   raw_srefrF   rr   rW   raw_trefrH   rw   rz   rs   	dedup_keyr\   raw_mrefrJ   r   r   	canonicalrk   ru   s-                                              @@r   	normalizer   _   s`   C=D)+L')J)+L')J ')O$&M&(O $&L!#J#%L!#J(,

51!SV)5J5!"aQ!LM * .m,-$quuW~quuV}=>Dz!2AEE,4G|I\^_^c^cdp^qrs$quuXj0A155CTUV 12!$$se~"eH:&'OTOQUV*%*%"Hc,/1#67H"$ w+!   	'A&LO	' EE$K|!4aeeM6JK 	.C#A%-"	.9.D (^  ,m,-T!%%"5quu=P7QRS6>_((2D
$quuW~quuV}aeeI>NOPd155#6l8KQUUSW[YZ
U 34!$$se~"eH:&'K4KTR(#q(#"Hc*oa/45H"((w+   	%A$JqM	% EE$Kz!2AEE-4HI 	,C#A#+a 	,; ,F *  .m,-QUU<01QUU:./6>_((2D
2:=$$X.d155#5quuY7G{I[\]	aeeFmQUU6]AEE)<LMNd155;l(;QUU;=OPQ(d3%LeL*hPY[]_c9d,e	$*%*%#I.Hc,/1#67H&.L#"($&w+	! 	 EE$K|!4aeeM6JK 	.C#A%-"	.; .F (^ #m,-QUU<01QUU:./T!%%"5quu]7KLM6>_((2D
2:=$$X.6>_((2D
tAEE,$7vfVW
d155;l(;QUU;=OPQ&0y1A4&H!%%	"WWYO
Q! 4 +AqDO
 (d3%L_L*jRTV]9^,_	
"(#q(##j/!+C01 (
9$ $$	
 		5#N   12?i$J$-j/-j/	! +
 <->?):;<->?):;!I$ G 6pO
s   bb$'b$c                  `   t        j                  d      } | j                  dddd       | j                  dd	dd
       | j                  ddd       | j                         }t	        |j
                  dd      5 }t        j                  |      }d d d        t              }t	        |j                  dd      5 }|j                  r+t        j                  ||dd       |j                  d       nt        j                  ||dd       d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nz;Normalize raw JSON into canonical entities with basic dedup)descriptionz--inputz-iTzPath to raw input JSON)requiredhelpz--outputz-ozPath to canonical output JSONz--pretty
store_truezPretty print JSON output)actionr   r   r1   )encodingwF   )r+   indent
),:)r+   
separators)argparseArgumentParseradd_argument
parse_argsopeninputr,   loadr   outputprettydumpwrite)parserargsfr?   r   s        r   mainr     s   $$1noF
	4$=UV

D4>]^

<>XYD	djj#	0 Aiil #I	dkk3	1 OQ;;IIiqAGGDMIIi:NO O 
O Os   =D>AD$D!$D-__main__)r
   N)r   rN   r4   r,   r   r   typingr   r   r   r   r	   r   r   r   r%   r9   r>   r^   r   r   __name__r   r   r   <module>r      s        ' 3 3I# I #   ?S	 ?c ?
 # ,$sCx. ,T#tDcN/C*C%D ,^v4S> vd38n vrO( zF r   