| { |
| "schema_version": 1, |
| "format": "comfy_scaled_fp8_merged_text_encoder", |
| "scale_policy": "preserve_stock_per_tensor_scale", |
| "base": "/workspace/Ideogram/checkpoints/text_encoders/qwen3vl_8b_fp8_scaled.safetensors", |
| "base_sha256": "4ba424cf62e51392e4d1a39933e803706f4e823c1065f36aaf149c6453f66bcd", |
| "adapter": "/workspace/Ideogram/artifacts/phase2_balanced_4000_8state_debanner_s020_v1_checkpoints/step_00002000/adapters.safetensors", |
| "adapter_sha256": "ade88e4b6928ca2556a868b566e7bccd3d927e80a0d3847ebd4583fab10bc63d", |
| "adapter_global_step": "2000", |
| "output": "/workspace/Ideogram/artifacts/huggingface_release/s020_v1/qwen3vl_8b_ideogram4_nl_s020_v1_step_00002000_fp8_scaled.safetensors", |
| "output_bytes": 10588637800, |
| "output_sha256": "5c505aec1b9aacac4483af9b09a0d3cd2d62a54b15ae3afbe83cf7945012eb98", |
| "output_tensors": 1254, |
| "merged_projections": 252, |
| "lora_rank": 64, |
| "lora_alpha": 64.0, |
| "elapsed_seconds": 61.729, |
| "projections": [ |
| { |
| "index": 1, |
| "total": 252, |
| "projection": "layers.0.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4391, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.1295157799351236, |
| "retained_delta_l2": 2.8502838015288536, |
| "delta_retention_norm_ratio": 0.9107747018894857, |
| "delta_cosine": 0.7261043086648739, |
| "relative_weight_rmse": 0.012313553632253754 |
| }, |
| { |
| "index": 2, |
| "total": 252, |
| "projection": "layers.0.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3781, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.308858942324612, |
| "retained_delta_l2": 3.0219341535998163, |
| "delta_retention_norm_ratio": 0.9132858808048134, |
| "delta_cosine": 0.7258020356483821, |
| "relative_weight_rmse": 0.013496873111315233 |
| }, |
| { |
| "index": 3, |
| "total": 252, |
| "projection": "layers.0.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.38, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.9333920986669484, |
| "retained_delta_l2": 2.621853415304213, |
| "delta_retention_norm_ratio": 0.893795758328963, |
| "delta_cosine": 0.7068022139752237, |
| "relative_weight_rmse": 0.01302276820099369 |
| }, |
| { |
| "index": 4, |
| "total": 252, |
| "projection": "layers.0.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0304, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.602349856468238, |
| "retained_delta_l2": 0.44131584785155675, |
| "delta_retention_norm_ratio": 0.7326570150429302, |
| "delta_cosine": 0.560430668071971, |
| "relative_weight_rmse": 0.009007892007776262 |
| }, |
| { |
| "index": 5, |
| "total": 252, |
| "projection": "layers.0.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1278, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9730080764510292, |
| "retained_delta_l2": 1.8610741013841146, |
| "delta_retention_norm_ratio": 0.9432673508015957, |
| "delta_cosine": 0.762789735772714, |
| "relative_weight_rmse": 0.013622010741295177 |
| }, |
| { |
| "index": 6, |
| "total": 252, |
| "projection": "layers.0.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.127, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.223923628071747, |
| "retained_delta_l2": 0.9518550472806543, |
| "delta_retention_norm_ratio": 0.7777078777213182, |
| "delta_cosine": 0.598442627196136, |
| "relative_weight_rmse": 0.009797371165960555 |
| }, |
| { |
| "index": 7, |
| "total": 252, |
| "projection": "layers.0.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0291, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.7957482199785566, |
| "retained_delta_l2": 0.6933537877647481, |
| "delta_retention_norm_ratio": 0.8713230772711401, |
| "delta_cosine": 0.6841275851947753, |
| "relative_weight_rmse": 0.011987200008821542 |
| }, |
| { |
| "index": 8, |
| "total": 252, |
| "projection": "layers.1.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4182, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.0362647955657387, |
| "retained_delta_l2": 1.9007486242357023, |
| "delta_retention_norm_ratio": 0.933448649888196, |
| "delta_cosine": 0.7565510162524389, |
| "relative_weight_rmse": 0.011982820281728953 |
| }, |
| { |
| "index": 9, |
| "total": 252, |
| "projection": "layers.1.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3785, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.8319098718741618, |
| "retained_delta_l2": 3.798833927371201, |
| "delta_retention_norm_ratio": 0.9913682874574543, |
| "delta_cosine": 0.8459146309675251, |
| "relative_weight_rmse": 0.014360352713851662 |
| }, |
| { |
| "index": 10, |
| "total": 252, |
| "projection": "layers.1.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3812, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.233617448400264, |
| "retained_delta_l2": 4.23070853812047, |
| "delta_retention_norm_ratio": 0.9993129019531762, |
| "delta_cosine": 0.929609242827706, |
| "relative_weight_rmse": 0.015762857213329554 |
| }, |
| { |
| "index": 11, |
| "total": 252, |
| "projection": "layers.1.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0305, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.6761311740026398, |
| "retained_delta_l2": 0.5551743744851045, |
| "delta_retention_norm_ratio": 0.8211045368586083, |
| "delta_cosine": 0.6373080054133655, |
| "relative_weight_rmse": 0.010814861671689595 |
| }, |
| { |
| "index": 12, |
| "total": 252, |
| "projection": "layers.1.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1282, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.7147328050292834, |
| "retained_delta_l2": 1.5341219402892992, |
| "delta_retention_norm_ratio": 0.8946711323127106, |
| "delta_cosine": 0.7053989373758582, |
| "relative_weight_rmse": 0.012667201785381724 |
| }, |
| { |
| "index": 13, |
| "total": 252, |
| "projection": "layers.1.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1276, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.0926542666331747, |
| "retained_delta_l2": 2.023345819293772, |
| "delta_retention_norm_ratio": 0.9668801251862249, |
| "delta_cosine": 0.7927101763251782, |
| "relative_weight_rmse": 0.014636375532536479 |
| }, |
| { |
| "index": 14, |
| "total": 252, |
| "projection": "layers.1.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0293, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9182738031034758, |
| "retained_delta_l2": 0.8358768448352694, |
| "delta_retention_norm_ratio": 0.910269727841815, |
| "delta_cosine": 0.7230853742697265, |
| "relative_weight_rmse": 0.012467732033120576 |
| }, |
| { |
| "index": 15, |
| "total": 252, |
| "projection": "layers.10.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4181, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.8113782214961516, |
| "retained_delta_l2": 2.492795325814636, |
| "delta_retention_norm_ratio": 0.8866808836870148, |
| "delta_cosine": 0.7006694785539181, |
| "relative_weight_rmse": 0.011660723877415962 |
| }, |
| { |
| "index": 16, |
| "total": 252, |
| "projection": "layers.10.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3776, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.388968981485157, |
| "retained_delta_l2": 1.8515130111340723, |
| "delta_retention_norm_ratio": 0.7750259737499969, |
| "delta_cosine": 0.5958672482894636, |
| "relative_weight_rmse": 0.009648177171664604 |
| }, |
| { |
| "index": 17, |
| "total": 252, |
| "projection": "layers.10.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3793, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.8546671366650447, |
| "retained_delta_l2": 2.525883959859384, |
| "delta_retention_norm_ratio": 0.8848260896751133, |
| "delta_cosine": 0.6973434819774882, |
| "relative_weight_rmse": 0.011903683508779728 |
| }, |
| { |
| "index": 18, |
| "total": 252, |
| "projection": "layers.10.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.031, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.898296856891836, |
| "retained_delta_l2": 0.8255322300406948, |
| "delta_retention_norm_ratio": 0.9189971262919572, |
| "delta_cosine": 0.7372096005027154, |
| "relative_weight_rmse": 0.011972097540133963 |
| }, |
| { |
| "index": 19, |
| "total": 252, |
| "projection": "layers.10.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1294, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.773726814891075, |
| "retained_delta_l2": 1.5849685945727472, |
| "delta_retention_norm_ratio": 0.8935810076649715, |
| "delta_cosine": 0.7068052779610151, |
| "relative_weight_rmse": 0.012165208402044565 |
| }, |
| { |
| "index": 20, |
| "total": 252, |
| "projection": "layers.10.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1269, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.1181889970136627, |
| "retained_delta_l2": 2.020497366449768, |
| "delta_retention_norm_ratio": 0.9538796440253322, |
| "delta_cosine": 0.7798677582043131, |
| "relative_weight_rmse": 0.013449250140918962 |
| }, |
| { |
| "index": 21, |
| "total": 252, |
| "projection": "layers.10.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0289, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9094617335032259, |
| "retained_delta_l2": 0.8120646973699481, |
| "delta_retention_norm_ratio": 0.8929069442448044, |
| "delta_cosine": 0.7111358371007716, |
| "relative_weight_rmse": 0.011201089063293127 |
| }, |
| { |
| "index": 22, |
| "total": 252, |
| "projection": "layers.11.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4199, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6083400962198855, |
| "retained_delta_l2": 2.2326133808508537, |
| "delta_retention_norm_ratio": 0.8559517925160334, |
| "delta_cosine": 0.6705604386194571, |
| "relative_weight_rmse": 0.011045630389532286 |
| }, |
| { |
| "index": 23, |
| "total": 252, |
| "projection": "layers.11.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.384, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.7719414238376725, |
| "retained_delta_l2": 2.339405933084685, |
| "delta_retention_norm_ratio": 0.8439593683209384, |
| "delta_cosine": 0.659378819125765, |
| "relative_weight_rmse": 0.010807064801439835 |
| }, |
| { |
| "index": 24, |
| "total": 252, |
| "projection": "layers.11.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3814, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6728273375827234, |
| "retained_delta_l2": 2.2706054481938365, |
| "delta_retention_norm_ratio": 0.8495144509586422, |
| "delta_cosine": 0.6631438140090375, |
| "relative_weight_rmse": 0.011368344170365544 |
| }, |
| { |
| "index": 25, |
| "total": 252, |
| "projection": "layers.11.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0303, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1285426297564844, |
| "retained_delta_l2": 1.0856459717547504, |
| "delta_retention_norm_ratio": 0.9619893330826232, |
| "delta_cosine": 0.7870521772506621, |
| "relative_weight_rmse": 0.01365816659419093 |
| }, |
| { |
| "index": 26, |
| "total": 252, |
| "projection": "layers.11.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.128, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.6698245744642877, |
| "retained_delta_l2": 1.466318444183154, |
| "delta_retention_norm_ratio": 0.8781272395955592, |
| "delta_cosine": 0.693043112285495, |
| "relative_weight_rmse": 0.011724965342670564 |
| }, |
| { |
| "index": 27, |
| "total": 252, |
| "projection": "layers.11.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1296, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.4064437305620288, |
| "retained_delta_l2": 1.169695210234028, |
| "delta_retention_norm_ratio": 0.8316686866431593, |
| "delta_cosine": 0.6464514802859761, |
| "relative_weight_rmse": 0.010750990640851749 |
| }, |
| { |
| "index": 28, |
| "total": 252, |
| "projection": "layers.11.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0292, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9976261903719174, |
| "retained_delta_l2": 0.9075747276461306, |
| "delta_retention_norm_ratio": 0.9097342636000612, |
| "delta_cosine": 0.7270924978235055, |
| "relative_weight_rmse": 0.011884848481938047 |
| }, |
| { |
| "index": 29, |
| "total": 252, |
| "projection": "layers.12.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4185, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.5673767174764017, |
| "retained_delta_l2": 2.1877978222092476, |
| "delta_retention_norm_ratio": 0.8521530195848076, |
| "delta_cosine": 0.6667567491113927, |
| "relative_weight_rmse": 0.010833193858063861 |
| }, |
| { |
| "index": 30, |
| "total": 252, |
| "projection": "layers.12.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3797, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.2128146573103873, |
| "retained_delta_l2": 2.880827242553919, |
| "delta_retention_norm_ratio": 0.8966677352516836, |
| "delta_cosine": 0.7127643911365393, |
| "relative_weight_rmse": 0.011978474399065964 |
| }, |
| { |
| "index": 31, |
| "total": 252, |
| "projection": "layers.12.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3784, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.685915319766742, |
| "retained_delta_l2": 2.2893747931314667, |
| "delta_retention_norm_ratio": 0.8523629826610795, |
| "delta_cosine": 0.6661181577537216, |
| "relative_weight_rmse": 0.011217255943459098 |
| }, |
| { |
| "index": 32, |
| "total": 252, |
| "projection": "layers.12.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0302, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9323220288102295, |
| "retained_delta_l2": 0.8656498962907475, |
| "delta_retention_norm_ratio": 0.928488086241441, |
| "delta_cosine": 0.7477878360800111, |
| "relative_weight_rmse": 0.012288728180501179 |
| }, |
| { |
| "index": 33, |
| "total": 252, |
| "projection": "layers.12.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1282, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.7732730109339851, |
| "retained_delta_l2": 1.6063538404039202, |
| "delta_retention_norm_ratio": 0.9058694462156459, |
| "delta_cosine": 0.7218775372986568, |
| "relative_weight_rmse": 0.012030348575707556 |
| }, |
| { |
| "index": 34, |
| "total": 252, |
| "projection": "layers.12.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1279, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.8059135630192593, |
| "retained_delta_l2": 1.6509890099638174, |
| "delta_retention_norm_ratio": 0.914212642161883, |
| "delta_cosine": 0.7311857138038845, |
| "relative_weight_rmse": 0.012431793623686902 |
| }, |
| { |
| "index": 35, |
| "total": 252, |
| "projection": "layers.12.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0294, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0870296328963427, |
| "retained_delta_l2": 1.0239311844644483, |
| "delta_retention_norm_ratio": 0.9419533317930153, |
| "delta_cosine": 0.7659336051191952, |
| "relative_weight_rmse": 0.012192820958434733 |
| }, |
| { |
| "index": 36, |
| "total": 252, |
| "projection": "layers.13.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4212, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.344962195460705, |
| "retained_delta_l2": 1.9205837316330012, |
| "delta_retention_norm_ratio": 0.8190254560823196, |
| "delta_cosine": 0.6358079001534217, |
| "relative_weight_rmse": 0.010191678392086631 |
| }, |
| { |
| "index": 37, |
| "total": 252, |
| "projection": "layers.13.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.377, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.9876379632012293, |
| "retained_delta_l2": 2.6129843228546905, |
| "delta_retention_norm_ratio": 0.8745987147836679, |
| "delta_cosine": 0.6875034086966243, |
| "relative_weight_rmse": 0.011880131031540675 |
| }, |
| { |
| "index": 38, |
| "total": 252, |
| "projection": "layers.13.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3865, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.4021005717028983, |
| "retained_delta_l2": 1.9422196707182822, |
| "delta_retention_norm_ratio": 0.808550521821575, |
| "delta_cosine": 0.6251684158384617, |
| "relative_weight_rmse": 0.010350500532359528 |
| }, |
| { |
| "index": 39, |
| "total": 252, |
| "projection": "layers.13.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0305, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.3411620116922038, |
| "retained_delta_l2": 1.3268981283278296, |
| "delta_retention_norm_ratio": 0.9893645337103033, |
| "delta_cosine": 0.8345317232908486, |
| "relative_weight_rmse": 0.014773482348064967 |
| }, |
| { |
| "index": 40, |
| "total": 252, |
| "projection": "layers.13.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.132, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.7296016855603318, |
| "retained_delta_l2": 1.5499819642029007, |
| "delta_retention_norm_ratio": 0.8961496610132866, |
| "delta_cosine": 0.7118572199998348, |
| "relative_weight_rmse": 0.012260288668890474 |
| }, |
| { |
| "index": 41, |
| "total": 252, |
| "projection": "layers.13.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1281, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.4093637562418553, |
| "retained_delta_l2": 1.1732090915470472, |
| "delta_retention_norm_ratio": 0.8324388124436184, |
| "delta_cosine": 0.6470332479577902, |
| "relative_weight_rmse": 0.010748170228869565 |
| }, |
| { |
| "index": 42, |
| "total": 252, |
| "projection": "layers.13.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0292, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.026313557917772, |
| "retained_delta_l2": 0.9447760209279176, |
| "delta_retention_norm_ratio": 0.9205529963422863, |
| "delta_cosine": 0.7379792429350502, |
| "relative_weight_rmse": 0.01230172862322642 |
| }, |
| { |
| "index": 43, |
| "total": 252, |
| "projection": "layers.14.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4209, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.467721314345068, |
| "retained_delta_l2": 2.086833464404717, |
| "delta_retention_norm_ratio": 0.8456519997917843, |
| "delta_cosine": 0.661331921101294, |
| "relative_weight_rmse": 0.01051814884350571 |
| }, |
| { |
| "index": 44, |
| "total": 252, |
| "projection": "layers.14.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3765, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.7321969937263284, |
| "retained_delta_l2": 2.3171825631863587, |
| "delta_retention_norm_ratio": 0.8481023032040055, |
| "delta_cosine": 0.6614208547580835, |
| "relative_weight_rmse": 0.011356928376026088 |
| }, |
| { |
| "index": 45, |
| "total": 252, |
| "projection": "layers.14.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3807, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.9443813100266794, |
| "retained_delta_l2": 2.610028726180295, |
| "delta_retention_norm_ratio": 0.8864438574216615, |
| "delta_cosine": 0.7008685493193013, |
| "relative_weight_rmse": 0.011782602760381216 |
| }, |
| { |
| "index": 46, |
| "total": 252, |
| "projection": "layers.14.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0303, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.7284994577708277, |
| "retained_delta_l2": 0.618501899336792, |
| "delta_retention_norm_ratio": 0.8490080435054507, |
| "delta_cosine": 0.6659961735692724, |
| "relative_weight_rmse": 0.010714954869028445 |
| }, |
| { |
| "index": 47, |
| "total": 252, |
| "projection": "layers.14.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1282, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.5558381795834701, |
| "retained_delta_l2": 1.354539985578835, |
| "delta_retention_norm_ratio": 0.8706175252374081, |
| "delta_cosine": 0.6849474569724862, |
| "relative_weight_rmse": 0.011441734643697268 |
| }, |
| { |
| "index": 48, |
| "total": 252, |
| "projection": "layers.14.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1269, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.318720626493867, |
| "retained_delta_l2": 2.2602193255120104, |
| "delta_retention_norm_ratio": 0.9747700088085574, |
| "delta_cosine": 0.8055496838820395, |
| "relative_weight_rmse": 0.01391485494866503 |
| }, |
| { |
| "index": 49, |
| "total": 252, |
| "projection": "layers.14.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.029, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0174185656146237, |
| "retained_delta_l2": 0.944473320821553, |
| "delta_retention_norm_ratio": 0.9283036035920925, |
| "delta_cosine": 0.7457334422346245, |
| "relative_weight_rmse": 0.012384185749512564 |
| }, |
| { |
| "index": 50, |
| "total": 252, |
| "projection": "layers.15.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4199, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.371493147271087, |
| "retained_delta_l2": 1.9731258260588727, |
| "delta_retention_norm_ratio": 0.8320183544824401, |
| "delta_cosine": 0.6481631721173573, |
| "relative_weight_rmse": 0.010300907614621341 |
| }, |
| { |
| "index": 51, |
| "total": 252, |
| "projection": "layers.15.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3794, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.8697631793277374, |
| "retained_delta_l2": 2.5091508929606365, |
| "delta_retention_norm_ratio": 0.874340750844961, |
| "delta_cosine": 0.6862452411628439, |
| "relative_weight_rmse": 0.0120082794787516 |
| }, |
| { |
| "index": 52, |
| "total": 252, |
| "projection": "layers.15.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3788, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.7874161707431244, |
| "retained_delta_l2": 2.438701622275863, |
| "delta_retention_norm_ratio": 0.874896848153717, |
| "delta_cosine": 0.689275565248844, |
| "relative_weight_rmse": 0.011420313770719347 |
| }, |
| { |
| "index": 53, |
| "total": 252, |
| "projection": "layers.15.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0304, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.8128830019881568, |
| "retained_delta_l2": 0.7186374562350791, |
| "delta_retention_norm_ratio": 0.8840601347025697, |
| "delta_cosine": 0.6988000834000759, |
| "relative_weight_rmse": 0.011576737232218115 |
| }, |
| { |
| "index": 54, |
| "total": 252, |
| "projection": "layers.15.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1271, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.150099775903978, |
| "retained_delta_l2": 2.0663030243918636, |
| "delta_retention_norm_ratio": 0.9610265753937475, |
| "delta_cosine": 0.7885854934462535, |
| "relative_weight_rmse": 0.013839239409670728 |
| }, |
| { |
| "index": 55, |
| "total": 252, |
| "projection": "layers.15.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1272, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.2362415876696247, |
| "retained_delta_l2": 2.1611246850921555, |
| "delta_retention_norm_ratio": 0.9664093079246648, |
| "delta_cosine": 0.801026727693609, |
| "relative_weight_rmse": 0.013447972564126687 |
| }, |
| { |
| "index": 56, |
| "total": 252, |
| "projection": "layers.15.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.029, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.855355924267693, |
| "retained_delta_l2": 0.7597572597045174, |
| "delta_retention_norm_ratio": 0.8882352224952182, |
| "delta_cosine": 0.704667737659905, |
| "relative_weight_rmse": 0.011198650304465709 |
| }, |
| { |
| "index": 57, |
| "total": 252, |
| "projection": "layers.16.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4184, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.490608050723674, |
| "retained_delta_l2": 2.1276880125739366, |
| "delta_retention_norm_ratio": 0.8542845639464277, |
| "delta_cosine": 0.6690464044195373, |
| "relative_weight_rmse": 0.010663801946869307 |
| }, |
| { |
| "index": 58, |
| "total": 252, |
| "projection": "layers.16.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3881, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6372762406162753, |
| "retained_delta_l2": 2.253994849358787, |
| "delta_retention_norm_ratio": 0.854667711575059, |
| "delta_cosine": 0.6681974156909103, |
| "relative_weight_rmse": 0.011063871539140003 |
| }, |
| { |
| "index": 59, |
| "total": 252, |
| "projection": "layers.16.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.38, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.5562915910917727, |
| "retained_delta_l2": 2.154870300649257, |
| "delta_retention_norm_ratio": 0.8429673313320756, |
| "delta_cosine": 0.6570978221314555, |
| "relative_weight_rmse": 0.01088591802268865 |
| }, |
| { |
| "index": 60, |
| "total": 252, |
| "projection": "layers.16.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0307, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.7402672696675293, |
| "retained_delta_l2": 0.6376569454010689, |
| "delta_retention_norm_ratio": 0.8613874630543302, |
| "delta_cosine": 0.675448610838514, |
| "relative_weight_rmse": 0.011215528271285295 |
| }, |
| { |
| "index": 61, |
| "total": 252, |
| "projection": "layers.16.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1304, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.6665277447725417, |
| "retained_delta_l2": 1.4904262957381975, |
| "delta_retention_norm_ratio": 0.8943303226803586, |
| "delta_cosine": 0.7094085759432628, |
| "relative_weight_rmse": 0.01193810158565392 |
| }, |
| { |
| "index": 62, |
| "total": 252, |
| "projection": "layers.16.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1275, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.8094508759468877, |
| "retained_delta_l2": 1.677429719498606, |
| "delta_retention_norm_ratio": 0.9270379990950598, |
| "delta_cosine": 0.7479558975300578, |
| "relative_weight_rmse": 0.012459277798183784 |
| }, |
| { |
| "index": 63, |
| "total": 252, |
| "projection": "layers.16.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0295, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9588999684784266, |
| "retained_delta_l2": 0.8758522584149606, |
| "delta_retention_norm_ratio": 0.9133927283413668, |
| "delta_cosine": 0.7302107892117504, |
| "relative_weight_rmse": 0.012150515736783439 |
| }, |
| { |
| "index": 64, |
| "total": 252, |
| "projection": "layers.17.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4182, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.57867394869486, |
| "retained_delta_l2": 2.217903125991572, |
| "delta_retention_norm_ratio": 0.8600944400567259, |
| "delta_cosine": 0.6745827781840225, |
| "relative_weight_rmse": 0.010922727095506776 |
| }, |
| { |
| "index": 65, |
| "total": 252, |
| "projection": "layers.17.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3773, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.530974860487244, |
| "retained_delta_l2": 3.309138766374794, |
| "delta_retention_norm_ratio": 0.9371742640835912, |
| "delta_cosine": 0.7559309585248899, |
| "relative_weight_rmse": 0.013279111898597819 |
| }, |
| { |
| "index": 66, |
| "total": 252, |
| "projection": "layers.17.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3772, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.182246137690255, |
| "retained_delta_l2": 2.8980044926585204, |
| "delta_retention_norm_ratio": 0.9106789252832457, |
| "delta_cosine": 0.725261046959691, |
| "relative_weight_rmse": 0.012372349212028403 |
| }, |
| { |
| "index": 67, |
| "total": 252, |
| "projection": "layers.17.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0306, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.7856562521842315, |
| "retained_delta_l2": 0.7028263759527129, |
| "delta_retention_norm_ratio": 0.8945723705485189, |
| "delta_cosine": 0.7142405090358148, |
| "relative_weight_rmse": 0.011541983384061129 |
| }, |
| { |
| "index": 68, |
| "total": 252, |
| "projection": "layers.17.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1281, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.7288860895233422, |
| "retained_delta_l2": 1.5687744025872319, |
| "delta_retention_norm_ratio": 0.9073902624896164, |
| "delta_cosine": 0.7245259556841699, |
| "relative_weight_rmse": 0.012197539719324683 |
| }, |
| { |
| "index": 69, |
| "total": 252, |
| "projection": "layers.17.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1276, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.659428444075962, |
| "retained_delta_l2": 1.4844032143221468, |
| "delta_retention_norm_ratio": 0.8945267990441875, |
| "delta_cosine": 0.7074773036620385, |
| "relative_weight_rmse": 0.011963215771788813 |
| }, |
| { |
| "index": 70, |
| "total": 252, |
| "projection": "layers.17.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0294, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.033762917861183, |
| "retained_delta_l2": 0.9695695450654802, |
| "delta_retention_norm_ratio": 0.9379031964809529, |
| "delta_cosine": 0.7590274849022801, |
| "relative_weight_rmse": 0.012764403414386711 |
| }, |
| { |
| "index": 71, |
| "total": 252, |
| "projection": "layers.18.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4196, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.910789276888511, |
| "retained_delta_l2": 2.6222802085342862, |
| "delta_retention_norm_ratio": 0.9008828737123057, |
| "delta_cosine": 0.7154197570893368, |
| "relative_weight_rmse": 0.011715650325460434 |
| }, |
| { |
| "index": 72, |
| "total": 252, |
| "projection": "layers.18.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3795, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.722324488785237, |
| "retained_delta_l2": 4.68356445391355, |
| "delta_retention_norm_ratio": 0.9917921703678483, |
| "delta_cosine": 0.8270502288163069, |
| "relative_weight_rmse": 0.015419870967505798 |
| }, |
| { |
| "index": 73, |
| "total": 252, |
| "projection": "layers.18.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3803, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.267632846087272, |
| "retained_delta_l2": 3.0065344734979704, |
| "delta_retention_norm_ratio": 0.9200955600314932, |
| "delta_cosine": 0.7366907817166277, |
| "relative_weight_rmse": 0.012487845832375657 |
| }, |
| { |
| "index": 74, |
| "total": 252, |
| "projection": "layers.18.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0304, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1178209584979375, |
| "retained_delta_l2": 1.0843822621846049, |
| "delta_retention_norm_ratio": 0.9700858209365965, |
| "delta_cosine": 0.7990006485228375, |
| "relative_weight_rmse": 0.014000429949694236 |
| }, |
| { |
| "index": 75, |
| "total": 252, |
| "projection": "layers.18.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1282, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.6294500157772767, |
| "retained_delta_l2": 1.4565664018421354, |
| "delta_retention_norm_ratio": 0.8939006337959542, |
| "delta_cosine": 0.7108714273975982, |
| "relative_weight_rmse": 0.011701311661202553 |
| }, |
| { |
| "index": 76, |
| "total": 252, |
| "projection": "layers.18.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1303, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.099352970985407, |
| "retained_delta_l2": 2.009225849712284, |
| "delta_retention_norm_ratio": 0.957069095802971, |
| "delta_cosine": 0.7836883856589295, |
| "relative_weight_rmse": 0.013542395647038019 |
| }, |
| { |
| "index": 77, |
| "total": 252, |
| "projection": "layers.18.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0291, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9281397272835645, |
| "retained_delta_l2": 0.8453631539465025, |
| "delta_retention_norm_ratio": 0.9108145348122006, |
| "delta_cosine": 0.7279251236491474, |
| "relative_weight_rmse": 0.011786639770044518 |
| }, |
| { |
| "index": 78, |
| "total": 252, |
| "projection": "layers.19.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4218, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6906724529290753, |
| "retained_delta_l2": 2.353773769212275, |
| "delta_retention_norm_ratio": 0.8747901539074179, |
| "delta_cosine": 0.6890261111788295, |
| "relative_weight_rmse": 0.011201229115173 |
| }, |
| { |
| "index": 79, |
| "total": 252, |
| "projection": "layers.19.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.379, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.6011590568420395, |
| "retained_delta_l2": 4.5324392054321665, |
| "delta_retention_norm_ratio": 0.9850646651070049, |
| "delta_cosine": 0.8179573786954442, |
| "relative_weight_rmse": 0.015420800468068915 |
| }, |
| { |
| "index": 80, |
| "total": 252, |
| "projection": "layers.19.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3787, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.088979201522636, |
| "retained_delta_l2": 2.7691085642458395, |
| "delta_retention_norm_ratio": 0.8964477853657532, |
| "delta_cosine": 0.7106646889590305, |
| "relative_weight_rmse": 0.012187361703435885 |
| }, |
| { |
| "index": 81, |
| "total": 252, |
| "projection": "layers.19.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0305, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0424679931914167, |
| "retained_delta_l2": 1.0031846977199295, |
| "delta_retention_norm_ratio": 0.9623170248601829, |
| "delta_cosine": 0.7948003347645736, |
| "relative_weight_rmse": 0.01357524445006692 |
| }, |
| { |
| "index": 82, |
| "total": 252, |
| "projection": "layers.19.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1285, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.6393473573479356, |
| "retained_delta_l2": 1.4597608928780879, |
| "delta_retention_norm_ratio": 0.8904524634971964, |
| "delta_cosine": 0.7051363882562122, |
| "relative_weight_rmse": 0.011792228026413542 |
| }, |
| { |
| "index": 83, |
| "total": 252, |
| "projection": "layers.19.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.128, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9324947526748775, |
| "retained_delta_l2": 1.8224312290030995, |
| "delta_retention_norm_ratio": 0.9430458874367277, |
| "delta_cosine": 0.765239868235387, |
| "relative_weight_rmse": 0.012807740543361366 |
| }, |
| { |
| "index": 84, |
| "total": 252, |
| "projection": "layers.19.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0294, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.2393259702908999, |
| "retained_delta_l2": 1.2060395661274452, |
| "delta_retention_norm_ratio": 0.973141526150992, |
| "delta_cosine": 0.806109674730439, |
| "relative_weight_rmse": 0.013900123324460853 |
| }, |
| { |
| "index": 85, |
| "total": 252, |
| "projection": "layers.2.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4212, |
| "parameters": 50331648, |
| "reference_delta_l2": 1.550514040883994, |
| "retained_delta_l2": 1.3354703881576557, |
| "delta_retention_norm_ratio": 0.8613081551949471, |
| "delta_cosine": 0.6980816143317669, |
| "relative_weight_rmse": 0.010102768254166766 |
| }, |
| { |
| "index": 86, |
| "total": 252, |
| "projection": "layers.2.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3786, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.5024354983712715, |
| "retained_delta_l2": 2.3585423713079154, |
| "delta_retention_norm_ratio": 0.942498766838542, |
| "delta_cosine": 0.7967290998301102, |
| "relative_weight_rmse": 0.012645736077248214 |
| }, |
| { |
| "index": 87, |
| "total": 252, |
| "projection": "layers.2.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3774, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.369649175371511, |
| "retained_delta_l2": 3.3497295641104907, |
| "delta_retention_norm_ratio": 0.9940885207259524, |
| "delta_cosine": 0.8793330738810422, |
| "relative_weight_rmse": 0.01495990024596177 |
| }, |
| { |
| "index": 88, |
| "total": 252, |
| "projection": "layers.2.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0305, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.5772932706105626, |
| "retained_delta_l2": 0.4344806465765585, |
| "delta_retention_norm_ratio": 0.7526168564498228, |
| "delta_cosine": 0.5763063069305538, |
| "relative_weight_rmse": 0.009291145467617177 |
| }, |
| { |
| "index": 89, |
| "total": 252, |
| "projection": "layers.2.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1283, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.0861834258535827, |
| "retained_delta_l2": 1.9678414833336078, |
| "delta_retention_norm_ratio": 0.9432734719999254, |
| "delta_cosine": 0.7605381226137655, |
| "relative_weight_rmse": 0.013949476644897577 |
| }, |
| { |
| "index": 90, |
| "total": 252, |
| "projection": "layers.2.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1273, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.2282960127488745, |
| "retained_delta_l2": 0.9700631471429637, |
| "delta_retention_norm_ratio": 0.7897633282811066, |
| "delta_cosine": 0.6079123164782136, |
| "relative_weight_rmse": 0.010177397449548601 |
| }, |
| { |
| "index": 91, |
| "total": 252, |
| "projection": "layers.2.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.03, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1607808521417493, |
| "retained_delta_l2": 1.1195868281071086, |
| "delta_retention_norm_ratio": 0.9645117991405234, |
| "delta_cosine": 0.793185053055849, |
| "relative_weight_rmse": 0.01351105097316285 |
| }, |
| { |
| "index": 92, |
| "total": 252, |
| "projection": "layers.20.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4246, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.64584748451423, |
| "retained_delta_l2": 2.258222891237714, |
| "delta_retention_norm_ratio": 0.8534970002824321, |
| "delta_cosine": 0.6658774979676224, |
| "relative_weight_rmse": 0.011247267660759373 |
| }, |
| { |
| "index": 93, |
| "total": 252, |
| "projection": "layers.20.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3789, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.526615232031967, |
| "retained_delta_l2": 4.455530846754728, |
| "delta_retention_norm_ratio": 0.984296349118825, |
| "delta_cosine": 0.8160620069203229, |
| "relative_weight_rmse": 0.01538652110350443 |
| }, |
| { |
| "index": 94, |
| "total": 252, |
| "projection": "layers.20.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3776, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.1237101903647013, |
| "retained_delta_l2": 2.806044559157967, |
| "delta_retention_norm_ratio": 0.8983050245228908, |
| "delta_cosine": 0.7120996555848715, |
| "relative_weight_rmse": 0.012270615442135124 |
| }, |
| { |
| "index": 95, |
| "total": 252, |
| "projection": "layers.20.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0305, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.8431576555668275, |
| "retained_delta_l2": 0.7751642730249042, |
| "delta_retention_norm_ratio": 0.9193586370318685, |
| "delta_cosine": 0.739759292876534, |
| "relative_weight_rmse": 0.012192669685611647 |
| }, |
| { |
| "index": 96, |
| "total": 252, |
| "projection": "layers.20.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1283, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.6098180721363975, |
| "retained_delta_l2": 1.428732374756382, |
| "delta_retention_norm_ratio": 0.8875117005366353, |
| "delta_cosine": 0.7034375530109004, |
| "relative_weight_rmse": 0.011826447278611532 |
| }, |
| { |
| "index": 97, |
| "total": 252, |
| "projection": "layers.20.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.128, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.8915653169770341, |
| "retained_delta_l2": 1.7676069670292385, |
| "delta_retention_norm_ratio": 0.9344678458442571, |
| "delta_cosine": 0.7568589852551876, |
| "relative_weight_rmse": 0.012334276756101584 |
| }, |
| { |
| "index": 98, |
| "total": 252, |
| "projection": "layers.20.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0294, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9440685914528588, |
| "retained_delta_l2": 0.8662821368972096, |
| "delta_retention_norm_ratio": 0.9176050816011778, |
| "delta_cosine": 0.7369546341937735, |
| "relative_weight_rmse": 0.012020241368954576 |
| }, |
| { |
| "index": 99, |
| "total": 252, |
| "projection": "layers.21.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4276, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.105774206302042, |
| "retained_delta_l2": 2.8037671599278715, |
| "delta_retention_norm_ratio": 0.9027594968876499, |
| "delta_cosine": 0.7157717939258454, |
| "relative_weight_rmse": 0.012307599987023648 |
| }, |
| { |
| "index": 100, |
| "total": 252, |
| "projection": "layers.21.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3775, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.603324023865813, |
| "retained_delta_l2": 4.538266389340112, |
| "delta_retention_norm_ratio": 0.9858672484951284, |
| "delta_cosine": 0.8160356108217762, |
| "relative_weight_rmse": 0.015878057128781025 |
| }, |
| { |
| "index": 101, |
| "total": 252, |
| "projection": "layers.21.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3776, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.339274323389295, |
| "retained_delta_l2": 3.0655218952248084, |
| "delta_retention_norm_ratio": 0.9180203835764432, |
| "delta_cosine": 0.7329672214327478, |
| "relative_weight_rmse": 0.012754384253425642 |
| }, |
| { |
| "index": 102, |
| "total": 252, |
| "projection": "layers.21.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0305, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.8880355259681696, |
| "retained_delta_l2": 0.8209643352818649, |
| "delta_retention_norm_ratio": 0.9244724014693204, |
| "delta_cosine": 0.7418171289261559, |
| "relative_weight_rmse": 0.01281140542251685 |
| }, |
| { |
| "index": 103, |
| "total": 252, |
| "projection": "layers.21.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1279, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.7371948241009854, |
| "retained_delta_l2": 1.58872774854642, |
| "delta_retention_norm_ratio": 0.9145363125109479, |
| "delta_cosine": 0.7322945496835428, |
| "relative_weight_rmse": 0.012386180007419076 |
| }, |
| { |
| "index": 104, |
| "total": 252, |
| "projection": "layers.21.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1263, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.345556330912787, |
| "retained_delta_l2": 1.1124933541307664, |
| "delta_retention_norm_ratio": 0.8267906207806868, |
| "delta_cosine": 0.6435156791814104, |
| "relative_weight_rmse": 0.010325715461482674 |
| }, |
| { |
| "index": 105, |
| "total": 252, |
| "projection": "layers.21.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0295, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.3430392811254306, |
| "retained_delta_l2": 1.3264946568200315, |
| "delta_retention_norm_ratio": 0.9876812059498847, |
| "delta_cosine": 0.8236209766657606, |
| "relative_weight_rmse": 0.014574284287276256 |
| }, |
| { |
| "index": 106, |
| "total": 252, |
| "projection": "layers.22.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4185, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.3563104342236816, |
| "retained_delta_l2": 3.1200426688610725, |
| "delta_retention_norm_ratio": 0.9296049128967839, |
| "delta_cosine": 0.7472098403303402, |
| "relative_weight_rmse": 0.012595485823080576 |
| }, |
| { |
| "index": 107, |
| "total": 252, |
| "projection": "layers.22.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.377, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.207905625812568, |
| "retained_delta_l2": 4.092846670489135, |
| "delta_retention_norm_ratio": 0.9726564791240501, |
| "delta_cosine": 0.8018039373008636, |
| "relative_weight_rmse": 0.01464523430893868 |
| }, |
| { |
| "index": 108, |
| "total": 252, |
| "projection": "layers.22.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3767, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.4246106949522344, |
| "retained_delta_l2": 3.173732205832326, |
| "delta_retention_norm_ratio": 0.926742479228458, |
| "delta_cosine": 0.7448358030341105, |
| "relative_weight_rmse": 0.01261764380333146 |
| }, |
| { |
| "index": 109, |
| "total": 252, |
| "projection": "layers.22.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0303, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1250380548192516, |
| "retained_delta_l2": 1.099810524114935, |
| "delta_retention_norm_ratio": 0.9775762867786996, |
| "delta_cosine": 0.8176936058321127, |
| "relative_weight_rmse": 0.01441093779238604 |
| }, |
| { |
| "index": 110, |
| "total": 252, |
| "projection": "layers.22.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1272, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.0979665245617816, |
| "retained_delta_l2": 1.9974504967278541, |
| "delta_retention_norm_ratio": 0.9520888314197848, |
| "delta_cosine": 0.7727993917982432, |
| "relative_weight_rmse": 0.013730056442512895 |
| }, |
| { |
| "index": 111, |
| "total": 252, |
| "projection": "layers.22.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1284, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.869767544241714, |
| "retained_delta_l2": 1.7405896445630222, |
| "delta_retention_norm_ratio": 0.9309123211189978, |
| "delta_cosine": 0.7546608977847129, |
| "relative_weight_rmse": 0.01259283284123554 |
| }, |
| { |
| "index": 112, |
| "total": 252, |
| "projection": "layers.22.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0298, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1037279157423996, |
| "retained_delta_l2": 1.0516996683747628, |
| "delta_retention_norm_ratio": 0.9528613468722127, |
| "delta_cosine": 0.77769661979401, |
| "relative_weight_rmse": 0.013220168147444164 |
| }, |
| { |
| "index": 113, |
| "total": 252, |
| "projection": "layers.23.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4216, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.160235840871493, |
| "retained_delta_l2": 2.9006612353003223, |
| "delta_retention_norm_ratio": 0.9178622676782286, |
| "delta_cosine": 0.7351433392668709, |
| "relative_weight_rmse": 0.011972793198786804 |
| }, |
| { |
| "index": 114, |
| "total": 252, |
| "projection": "layers.23.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3778, |
| "parameters": 50331648, |
| "reference_delta_l2": 5.626287078192542, |
| "retained_delta_l2": 5.669300174595202, |
| "delta_retention_norm_ratio": 1.007645023406178, |
| "delta_cosine": 0.8530969607907128, |
| "relative_weight_rmse": 0.01679839814486996 |
| }, |
| { |
| "index": 115, |
| "total": 252, |
| "projection": "layers.23.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3775, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.980546573065696, |
| "retained_delta_l2": 4.943087972922342, |
| "delta_retention_norm_ratio": 0.9924790181973349, |
| "delta_cosine": 0.8288916075112348, |
| "relative_weight_rmse": 0.015410818804014629 |
| }, |
| { |
| "index": 116, |
| "total": 252, |
| "projection": "layers.23.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0301, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.7941298246634246, |
| "retained_delta_l2": 0.7066560415227385, |
| "delta_retention_norm_ratio": 0.8898495177690121, |
| "delta_cosine": 0.7054838997759174, |
| "relative_weight_rmse": 0.011970656431982295 |
| }, |
| { |
| "index": 117, |
| "total": 252, |
| "projection": "layers.23.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1327, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9616796991461982, |
| "retained_delta_l2": 1.834908647487434, |
| "delta_retention_norm_ratio": 0.9353762738565629, |
| "delta_cosine": 0.7548535319766523, |
| "relative_weight_rmse": 0.012811803614575604 |
| }, |
| { |
| "index": 118, |
| "total": 252, |
| "projection": "layers.23.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1304, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.7868156453388941, |
| "retained_delta_l2": 1.6194093744676163, |
| "delta_retention_norm_ratio": 0.9063102725186141, |
| "delta_cosine": 0.7240533048699785, |
| "relative_weight_rmse": 0.01202873511087103 |
| }, |
| { |
| "index": 119, |
| "total": 252, |
| "projection": "layers.23.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0293, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9269632845956063, |
| "retained_delta_l2": 0.8512550950173756, |
| "delta_retention_norm_ratio": 0.9183266577691274, |
| "delta_cosine": 0.7375855337175854, |
| "relative_weight_rmse": 0.011726307762073923 |
| }, |
| { |
| "index": 120, |
| "total": 252, |
| "projection": "layers.24.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4183, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.2514706960498443, |
| "retained_delta_l2": 2.9793935187526754, |
| "delta_retention_norm_ratio": 0.9163218116565803, |
| "delta_cosine": 0.7316018859300564, |
| "relative_weight_rmse": 0.01227576206795573 |
| }, |
| { |
| "index": 121, |
| "total": 252, |
| "projection": "layers.24.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3792, |
| "parameters": 50331648, |
| "reference_delta_l2": 5.11220125208371, |
| "retained_delta_l2": 5.0793680797612115, |
| "delta_retention_norm_ratio": 0.9935774883062133, |
| "delta_cosine": 0.8268587806603359, |
| "relative_weight_rmse": 0.01622681210023664 |
| }, |
| { |
| "index": 122, |
| "total": 252, |
| "projection": "layers.24.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3823, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.008939223739621, |
| "retained_delta_l2": 3.8306350545945733, |
| "delta_retention_norm_ratio": 0.9555233543853223, |
| "delta_cosine": 0.7755895473310437, |
| "relative_weight_rmse": 0.013947869007459165 |
| }, |
| { |
| "index": 123, |
| "total": 252, |
| "projection": "layers.24.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0306, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9657699046880037, |
| "retained_delta_l2": 0.9163237914003645, |
| "delta_retention_norm_ratio": 0.9488013521154264, |
| "delta_cosine": 0.7723764319730205, |
| "relative_weight_rmse": 0.013146854028355406 |
| }, |
| { |
| "index": 124, |
| "total": 252, |
| "projection": "layers.24.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1284, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9180386974565615, |
| "retained_delta_l2": 1.778889888553453, |
| "delta_retention_norm_ratio": 0.9274525539616961, |
| "delta_cosine": 0.7461192695903845, |
| "relative_weight_rmse": 0.012580078475031757 |
| }, |
| { |
| "index": 125, |
| "total": 252, |
| "projection": "layers.24.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1279, |
| "parameters": 16777216, |
| "reference_delta_l2": 3.0223934430575605, |
| "retained_delta_l2": 3.019102590965159, |
| "delta_retention_norm_ratio": 0.998911176802623, |
| "delta_cosine": 0.8488570729763014, |
| "relative_weight_rmse": 0.015556414370022116 |
| }, |
| { |
| "index": 126, |
| "total": 252, |
| "projection": "layers.24.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0294, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0890605670331526, |
| "retained_delta_l2": 1.0379482359911925, |
| "delta_retention_norm_ratio": 0.9530675036915516, |
| "delta_cosine": 0.7793129266050705, |
| "relative_weight_rmse": 0.013058435495478861 |
| }, |
| { |
| "index": 127, |
| "total": 252, |
| "projection": "layers.25.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4228, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.63616379677842, |
| "retained_delta_l2": 3.4026856581495566, |
| "delta_retention_norm_ratio": 0.9357899831587012, |
| "delta_cosine": 0.75136838655561, |
| "relative_weight_rmse": 0.013261149732231414 |
| }, |
| { |
| "index": 128, |
| "total": 252, |
| "projection": "layers.25.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3792, |
| "parameters": 50331648, |
| "reference_delta_l2": 5.588137120575692, |
| "retained_delta_l2": 5.600545330822972, |
| "delta_retention_norm_ratio": 1.0022204555793008, |
| "delta_cosine": 0.8402136399093728, |
| "relative_weight_rmse": 0.01693661749294787 |
| }, |
| { |
| "index": 129, |
| "total": 252, |
| "projection": "layers.25.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3774, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.744055871537401, |
| "retained_delta_l2": 3.4942314485826187, |
| "delta_retention_norm_ratio": 0.9332743870480228, |
| "delta_cosine": 0.7482273397250873, |
| "relative_weight_rmse": 0.013618174422273666 |
| }, |
| { |
| "index": 130, |
| "total": 252, |
| "projection": "layers.25.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0307, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0283780121709922, |
| "retained_delta_l2": 0.9818159558303128, |
| "delta_retention_norm_ratio": 0.9547228200237546, |
| "delta_cosine": 0.779950811230599, |
| "relative_weight_rmse": 0.013540249177058672 |
| }, |
| { |
| "index": 131, |
| "total": 252, |
| "projection": "layers.25.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1276, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9550480162383896, |
| "retained_delta_l2": 1.8079483667265448, |
| "delta_retention_norm_ratio": 0.9247590604987431, |
| "delta_cosine": 0.7401215782827391, |
| "relative_weight_rmse": 0.013194625829935585 |
| }, |
| { |
| "index": 132, |
| "total": 252, |
| "projection": "layers.25.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1269, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.4471720032329207, |
| "retained_delta_l2": 2.399520063203793, |
| "delta_retention_norm_ratio": 0.9805277520475981, |
| "delta_cosine": 0.8149741021838156, |
| "relative_weight_rmse": 0.014585611942000051 |
| }, |
| { |
| "index": 133, |
| "total": 252, |
| "projection": "layers.25.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0289, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1872069387379602, |
| "retained_delta_l2": 1.1437759901729765, |
| "delta_retention_norm_ratio": 0.9634175415019455, |
| "delta_cosine": 0.7871225418419032, |
| "relative_weight_rmse": 0.013451197170588274 |
| }, |
| { |
| "index": 134, |
| "total": 252, |
| "projection": "layers.26.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4216, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.2948821444778953, |
| "retained_delta_l2": 3.0158551550364296, |
| "delta_retention_norm_ratio": 0.9153150318565705, |
| "delta_cosine": 0.728234076979134, |
| "relative_weight_rmse": 0.012335675013045194 |
| }, |
| { |
| "index": 135, |
| "total": 252, |
| "projection": "layers.26.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3819, |
| "parameters": 50331648, |
| "reference_delta_l2": 5.257820107805619, |
| "retained_delta_l2": 5.22001821197644, |
| "delta_retention_norm_ratio": 0.9928103481948612, |
| "delta_cosine": 0.8220782001203848, |
| "relative_weight_rmse": 0.01665213701027111 |
| }, |
| { |
| "index": 136, |
| "total": 252, |
| "projection": "layers.26.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3807, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.5989018785770184, |
| "retained_delta_l2": 3.3268782382218878, |
| "delta_retention_norm_ratio": 0.9244148216503509, |
| "delta_cosine": 0.738176113660347, |
| "relative_weight_rmse": 0.01316718412135077 |
| }, |
| { |
| "index": 137, |
| "total": 252, |
| "projection": "layers.26.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0307, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1017757878657917, |
| "retained_delta_l2": 1.0633556595201925, |
| "delta_retention_norm_ratio": 0.9651289048382327, |
| "delta_cosine": 0.7912559274637104, |
| "relative_weight_rmse": 0.014435964750473713 |
| }, |
| { |
| "index": 138, |
| "total": 252, |
| "projection": "layers.26.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1282, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.14452140481544, |
| "retained_delta_l2": 2.0199786558489006, |
| "delta_retention_norm_ratio": 0.9419251546350231, |
| "delta_cosine": 0.7603254744735352, |
| "relative_weight_rmse": 0.013806357027894066 |
| }, |
| { |
| "index": 139, |
| "total": 252, |
| "projection": "layers.26.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1278, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.7102624259983674, |
| "retained_delta_l2": 2.6858010439526647, |
| "delta_retention_norm_ratio": 0.9909745337532428, |
| "delta_cosine": 0.8318261324732998, |
| "relative_weight_rmse": 0.015352093328554573 |
| }, |
| { |
| "index": 140, |
| "total": 252, |
| "projection": "layers.26.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0292, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.3006259143024874, |
| "retained_delta_l2": 1.2651158277502554, |
| "delta_retention_norm_ratio": 0.9726976941165473, |
| "delta_cosine": 0.8067860578136268, |
| "relative_weight_rmse": 0.014257367480946508 |
| }, |
| { |
| "index": 141, |
| "total": 252, |
| "projection": "layers.27.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4172, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.1210244717525866, |
| "retained_delta_l2": 2.7939265449511432, |
| "delta_retention_norm_ratio": 0.8951953341725115, |
| "delta_cosine": 0.707603604841936, |
| "relative_weight_rmse": 0.011908033331055366 |
| }, |
| { |
| "index": 142, |
| "total": 252, |
| "projection": "layers.27.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.376, |
| "parameters": 50331648, |
| "reference_delta_l2": 5.223471568922569, |
| "retained_delta_l2": 5.190236864464552, |
| "delta_retention_norm_ratio": 0.9936374298166475, |
| "delta_cosine": 0.8248092087450305, |
| "relative_weight_rmse": 0.016444172947481323 |
| }, |
| { |
| "index": 143, |
| "total": 252, |
| "projection": "layers.27.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3804, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.1443452802142606, |
| "retained_delta_l2": 2.7614216127494564, |
| "delta_retention_norm_ratio": 0.8782183146760806, |
| "delta_cosine": 0.6899573747353942, |
| "relative_weight_rmse": 0.01218497507994894 |
| }, |
| { |
| "index": 144, |
| "total": 252, |
| "projection": "layers.27.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0301, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9579004907051812, |
| "retained_delta_l2": 0.9016451638898682, |
| "delta_retention_norm_ratio": 0.9412722643310273, |
| "delta_cosine": 0.7665693409053349, |
| "relative_weight_rmse": 0.013499582772046447 |
| }, |
| { |
| "index": 145, |
| "total": 252, |
| "projection": "layers.27.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1269, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.8332854378342283, |
| "retained_delta_l2": 1.6681613288420845, |
| "delta_retention_norm_ratio": 0.9099299511224967, |
| "delta_cosine": 0.7238126461520346, |
| "relative_weight_rmse": 0.012739701931374197 |
| }, |
| { |
| "index": 146, |
| "total": 252, |
| "projection": "layers.27.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1261, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.5066116371408538, |
| "retained_delta_l2": 2.4520825621119036, |
| "delta_retention_norm_ratio": 0.978245902069158, |
| "delta_cosine": 0.813475660318453, |
| "relative_weight_rmse": 0.01490029214231884 |
| }, |
| { |
| "index": 147, |
| "total": 252, |
| "projection": "layers.27.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0288, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.305228018543387, |
| "retained_delta_l2": 1.2820434572650723, |
| "delta_retention_norm_ratio": 0.9822371563060772, |
| "delta_cosine": 0.8141868320028673, |
| "relative_weight_rmse": 0.01431323852325054 |
| }, |
| { |
| "index": 148, |
| "total": 252, |
| "projection": "layers.28.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4181, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.7609601109774404, |
| "retained_delta_l2": 3.5458474311195634, |
| "delta_retention_norm_ratio": 0.9428037858657397, |
| "delta_cosine": 0.7609299525782532, |
| "relative_weight_rmse": 0.013099285885407297 |
| }, |
| { |
| "index": 149, |
| "total": 252, |
| "projection": "layers.28.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3767, |
| "parameters": 50331648, |
| "reference_delta_l2": 6.409505864458649, |
| "retained_delta_l2": 6.522239218789574, |
| "delta_retention_norm_ratio": 1.0175884626233112, |
| "delta_cosine": 0.8665051334261239, |
| "relative_weight_rmse": 0.017929193401937306 |
| }, |
| { |
| "index": 150, |
| "total": 252, |
| "projection": "layers.28.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3797, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.240790766573222, |
| "retained_delta_l2": 2.8726819439189053, |
| "delta_retention_norm_ratio": 0.8864138881006652, |
| "delta_cosine": 0.6976513684563926, |
| "relative_weight_rmse": 0.012317570764878012 |
| }, |
| { |
| "index": 151, |
| "total": 252, |
| "projection": "layers.28.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0301, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0315480582853214, |
| "retained_delta_l2": 0.9815406033974483, |
| "delta_retention_norm_ratio": 0.95152193396496, |
| "delta_cosine": 0.7709704314678171, |
| "relative_weight_rmse": 0.014130402020247729 |
| }, |
| { |
| "index": 152, |
| "total": 252, |
| "projection": "layers.28.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1271, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.866301512748793, |
| "retained_delta_l2": 1.714424115865921, |
| "delta_retention_norm_ratio": 0.9186211896387639, |
| "delta_cosine": 0.7336246512815289, |
| "relative_weight_rmse": 0.012828730649353547 |
| }, |
| { |
| "index": 153, |
| "total": 252, |
| "projection": "layers.28.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1276, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.1056798548733475, |
| "retained_delta_l2": 2.0057791667171596, |
| "delta_retention_norm_ratio": 0.9525565636556862, |
| "delta_cosine": 0.776883814777253, |
| "relative_weight_rmse": 0.013245033292334856 |
| }, |
| { |
| "index": 154, |
| "total": 252, |
| "projection": "layers.28.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0291, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0578716443620564, |
| "retained_delta_l2": 0.9885759113310327, |
| "delta_retention_norm_ratio": 0.9344951408799579, |
| "delta_cosine": 0.7520981682275366, |
| "relative_weight_rmse": 0.012589562464635296 |
| }, |
| { |
| "index": 155, |
| "total": 252, |
| "projection": "layers.29.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4172, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.7400414679280893, |
| "retained_delta_l2": 3.5093891045814924, |
| "delta_retention_norm_ratio": 0.9383289288836753, |
| "delta_cosine": 0.7558183147959916, |
| "relative_weight_rmse": 0.012999432967736499 |
| }, |
| { |
| "index": 156, |
| "total": 252, |
| "projection": "layers.29.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3765, |
| "parameters": 50331648, |
| "reference_delta_l2": 6.119080985772878, |
| "retained_delta_l2": 6.202541880165826, |
| "delta_retention_norm_ratio": 1.0136394492223584, |
| "delta_cosine": 0.8565729728321285, |
| "relative_weight_rmse": 0.017705159618673742 |
| }, |
| { |
| "index": 157, |
| "total": 252, |
| "projection": "layers.29.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.377, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.5792385362284014, |
| "retained_delta_l2": 3.278999355649104, |
| "delta_retention_norm_ratio": 0.9161164651251011, |
| "delta_cosine": 0.7293019135804754, |
| "relative_weight_rmse": 0.012895063583214437 |
| }, |
| { |
| "index": 158, |
| "total": 252, |
| "projection": "layers.29.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0335, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9611100218066246, |
| "retained_delta_l2": 0.9109171148597183, |
| "delta_retention_norm_ratio": 0.9477761069928733, |
| "delta_cosine": 0.7815760941041824, |
| "relative_weight_rmse": 0.013125099298455387 |
| }, |
| { |
| "index": 159, |
| "total": 252, |
| "projection": "layers.29.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1298, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9448868287407715, |
| "retained_delta_l2": 1.7931075348352579, |
| "delta_retention_norm_ratio": 0.9219598324886678, |
| "delta_cosine": 0.7375264672159623, |
| "relative_weight_rmse": 0.012926364794946505 |
| }, |
| { |
| "index": 160, |
| "total": 252, |
| "projection": "layers.29.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1282, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.9410678323592614, |
| "retained_delta_l2": 2.927741570929101, |
| "delta_retention_norm_ratio": 0.995468903748653, |
| "delta_cosine": 0.8476177653552707, |
| "relative_weight_rmse": 0.015271058137973317 |
| }, |
| { |
| "index": 161, |
| "total": 252, |
| "projection": "layers.29.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0291, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0694664804434102, |
| "retained_delta_l2": 1.010059414365901, |
| "delta_retention_norm_ratio": 0.9444516801939613, |
| "delta_cosine": 0.7675995472724292, |
| "relative_weight_rmse": 0.01295360202158742 |
| }, |
| { |
| "index": 162, |
| "total": 252, |
| "projection": "layers.3.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4203, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.0612252225954784, |
| "retained_delta_l2": 1.8231090160941608, |
| "delta_retention_norm_ratio": 0.8844783171236942, |
| "delta_cosine": 0.7151755493808815, |
| "relative_weight_rmse": 0.01132277695851018 |
| }, |
| { |
| "index": 163, |
| "total": 252, |
| "projection": "layers.3.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3757, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.9220544512009474, |
| "retained_delta_l2": 2.7004006597997616, |
| "delta_retention_norm_ratio": 0.9241445376522374, |
| "delta_cosine": 0.7457059707004465, |
| "relative_weight_rmse": 0.012247179531549551 |
| }, |
| { |
| "index": 164, |
| "total": 252, |
| "projection": "layers.3.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3801, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.1727358381363153, |
| "retained_delta_l2": 3.106383395466854, |
| "delta_retention_norm_ratio": 0.9790866791139986, |
| "delta_cosine": 0.8418774960196438, |
| "relative_weight_rmse": 0.014668988845837653 |
| }, |
| { |
| "index": 165, |
| "total": 252, |
| "projection": "layers.3.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0307, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.5945609896406548, |
| "retained_delta_l2": 0.4614519112115394, |
| "delta_retention_norm_ratio": 0.7761220787297786, |
| "delta_cosine": 0.5970850492537589, |
| "relative_weight_rmse": 0.009637125549736558 |
| }, |
| { |
| "index": 166, |
| "total": 252, |
| "projection": "layers.3.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1306, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.8551680918690623, |
| "retained_delta_l2": 1.702378465435281, |
| "delta_retention_norm_ratio": 0.9176410875632044, |
| "delta_cosine": 0.7310633062118554, |
| "relative_weight_rmse": 0.012836174509958146 |
| }, |
| { |
| "index": 167, |
| "total": 252, |
| "projection": "layers.3.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1283, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.50916759836078, |
| "retained_delta_l2": 1.3066576875077118, |
| "delta_retention_norm_ratio": 0.8658135046942239, |
| "delta_cosine": 0.6791651347451496, |
| "relative_weight_rmse": 0.011729254761278396 |
| }, |
| { |
| "index": 168, |
| "total": 252, |
| "projection": "layers.3.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0304, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9493558174419165, |
| "retained_delta_l2": 0.8560802908478509, |
| "delta_retention_norm_ratio": 0.9017486121848381, |
| "delta_cosine": 0.7160920598769267, |
| "relative_weight_rmse": 0.012411782534285076 |
| }, |
| { |
| "index": 169, |
| "total": 252, |
| "projection": "layers.30.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.426, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.891216371645432, |
| "retained_delta_l2": 3.691144031245628, |
| "delta_retention_norm_ratio": 0.9485835992422077, |
| "delta_cosine": 0.7667413673469646, |
| "relative_weight_rmse": 0.01314762686300376 |
| }, |
| { |
| "index": 170, |
| "total": 252, |
| "projection": "layers.30.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3781, |
| "parameters": 50331648, |
| "reference_delta_l2": 5.390923699257254, |
| "retained_delta_l2": 5.38683376050761, |
| "delta_retention_norm_ratio": 0.9992413287633423, |
| "delta_cosine": 0.8303979683713872, |
| "relative_weight_rmse": 0.016918480433160996 |
| }, |
| { |
| "index": 171, |
| "total": 252, |
| "projection": "layers.30.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3779, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.803511864260935, |
| "retained_delta_l2": 3.5221099707322403, |
| "delta_retention_norm_ratio": 0.9260152449706176, |
| "delta_cosine": 0.7392997590801647, |
| "relative_weight_rmse": 0.013392195073316317 |
| }, |
| { |
| "index": 172, |
| "total": 252, |
| "projection": "layers.30.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0304, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.493770964022566, |
| "retained_delta_l2": 1.5055196729336286, |
| "delta_retention_norm_ratio": 1.0078651340761267, |
| "delta_cosine": 0.8526237968690384, |
| "relative_weight_rmse": 0.01659810135995429 |
| }, |
| { |
| "index": 173, |
| "total": 252, |
| "projection": "layers.30.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.13, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.869066986144252, |
| "retained_delta_l2": 1.6929682622874866, |
| "delta_retention_norm_ratio": 0.9057825507794967, |
| "delta_cosine": 0.7176126199142076, |
| "relative_weight_rmse": 0.012864923697526695 |
| }, |
| { |
| "index": 174, |
| "total": 252, |
| "projection": "layers.30.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1274, |
| "parameters": 16777216, |
| "reference_delta_l2": 3.0153665146577913, |
| "retained_delta_l2": 3.026151166028698, |
| "delta_retention_norm_ratio": 1.0035765640158443, |
| "delta_cosine": 0.8548589463251729, |
| "relative_weight_rmse": 0.016286740512417037 |
| }, |
| { |
| "index": 175, |
| "total": 252, |
| "projection": "layers.30.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0293, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.3107439157647822, |
| "retained_delta_l2": 1.2728059144749695, |
| "delta_retention_norm_ratio": 0.9710561301612628, |
| "delta_cosine": 0.8016661063381451, |
| "relative_weight_rmse": 0.013927134335966412 |
| }, |
| { |
| "index": 176, |
| "total": 252, |
| "projection": "layers.31.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4218, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.112663100444478, |
| "retained_delta_l2": 3.947313056520835, |
| "delta_retention_norm_ratio": 0.9597948969110129, |
| "delta_cosine": 0.7806269001044362, |
| "relative_weight_rmse": 0.013383008011024313 |
| }, |
| { |
| "index": 177, |
| "total": 252, |
| "projection": "layers.31.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3774, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.207933372829771, |
| "retained_delta_l2": 4.058604717995024, |
| "delta_retention_norm_ratio": 0.9645125904799376, |
| "delta_cosine": 0.785362693443029, |
| "relative_weight_rmse": 0.014746817633539662 |
| }, |
| { |
| "index": 178, |
| "total": 252, |
| "projection": "layers.31.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3773, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.744293271609304, |
| "retained_delta_l2": 3.4567529672151363, |
| "delta_retention_norm_ratio": 0.9232057204027231, |
| "delta_cosine": 0.7366416220496778, |
| "relative_weight_rmse": 0.013122450308699104 |
| }, |
| { |
| "index": 179, |
| "total": 252, |
| "projection": "layers.31.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0304, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1306844138330658, |
| "retained_delta_l2": 1.1067167777558367, |
| "delta_retention_norm_ratio": 0.9788025413775909, |
| "delta_cosine": 0.8128010662192252, |
| "relative_weight_rmse": 0.014431528691286433 |
| }, |
| { |
| "index": 180, |
| "total": 252, |
| "projection": "layers.31.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1291, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9262812779313172, |
| "retained_delta_l2": 1.7774918803748536, |
| "delta_retention_norm_ratio": 0.9227582185109268, |
| "delta_cosine": 0.7387240864420156, |
| "relative_weight_rmse": 0.01253640903866733 |
| }, |
| { |
| "index": 181, |
| "total": 252, |
| "projection": "layers.31.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1277, |
| "parameters": 16777216, |
| "reference_delta_l2": 3.091232754030015, |
| "retained_delta_l2": 3.114182056358259, |
| "delta_retention_norm_ratio": 1.0074239968822551, |
| "delta_cosine": 0.859119072636568, |
| "relative_weight_rmse": 0.016349614371943485 |
| }, |
| { |
| "index": 182, |
| "total": 252, |
| "projection": "layers.31.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0293, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9129372319850592, |
| "retained_delta_l2": 0.80527012589888, |
| "delta_retention_norm_ratio": 0.8820651603264427, |
| "delta_cosine": 0.699342117861831, |
| "relative_weight_rmse": 0.011244243849305599 |
| }, |
| { |
| "index": 183, |
| "total": 252, |
| "projection": "layers.32.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4188, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.0531319642760333, |
| "retained_delta_l2": 2.700968335821406, |
| "delta_retention_norm_ratio": 0.884654960029501, |
| "delta_cosine": 0.6983479871907332, |
| "relative_weight_rmse": 0.011288085442385564 |
| }, |
| { |
| "index": 184, |
| "total": 252, |
| "projection": "layers.32.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3821, |
| "parameters": 50331648, |
| "reference_delta_l2": 5.724801736222787, |
| "retained_delta_l2": 5.7750467166699035, |
| "delta_retention_norm_ratio": 1.0087767197472008, |
| "delta_cosine": 0.8479988489236641, |
| "relative_weight_rmse": 0.017468866184904154 |
| }, |
| { |
| "index": 185, |
| "total": 252, |
| "projection": "layers.32.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3873, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.175474368127258, |
| "retained_delta_l2": 3.9589358467876394, |
| "delta_retention_norm_ratio": 0.9481403782543784, |
| "delta_cosine": 0.7671152769314903, |
| "relative_weight_rmse": 0.013828873112894892 |
| }, |
| { |
| "index": 186, |
| "total": 252, |
| "projection": "layers.32.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0343, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0750931486612259, |
| "retained_delta_l2": 1.040542228910182, |
| "delta_retention_norm_ratio": 0.9678623942547965, |
| "delta_cosine": 0.7986368381511688, |
| "relative_weight_rmse": 0.014081564546770174 |
| }, |
| { |
| "index": 187, |
| "total": 252, |
| "projection": "layers.32.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1296, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9535348950549298, |
| "retained_delta_l2": 1.7957168772677157, |
| "delta_retention_norm_ratio": 0.9192141291221846, |
| "delta_cosine": 0.7328067682184131, |
| "relative_weight_rmse": 0.012910728503809407 |
| }, |
| { |
| "index": 188, |
| "total": 252, |
| "projection": "layers.32.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.127, |
| "parameters": 16777216, |
| "reference_delta_l2": 3.498986324699088, |
| "retained_delta_l2": 3.558383258594726, |
| "delta_retention_norm_ratio": 1.0169754690026536, |
| "delta_cosine": 0.8835599014845256, |
| "relative_weight_rmse": 0.017005628509784897 |
| }, |
| { |
| "index": 189, |
| "total": 252, |
| "projection": "layers.32.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0293, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1058993601101823, |
| "retained_delta_l2": 1.0362872940572145, |
| "delta_retention_norm_ratio": 0.937053887031789, |
| "delta_cosine": 0.7581904018210112, |
| "relative_weight_rmse": 0.01261230357564126 |
| }, |
| { |
| "index": 190, |
| "total": 252, |
| "projection": "layers.33.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.42, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.680454662657774, |
| "retained_delta_l2": 4.59402019908065, |
| "delta_retention_norm_ratio": 0.9815328916084314, |
| "delta_cosine": 0.809854469246563, |
| "relative_weight_rmse": 0.01440706327110179 |
| }, |
| { |
| "index": 191, |
| "total": 252, |
| "projection": "layers.33.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3783, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.9928551759715663, |
| "retained_delta_l2": 3.8067651307265034, |
| "delta_retention_norm_ratio": 0.9533942412024042, |
| "delta_cosine": 0.7702773788126092, |
| "relative_weight_rmse": 0.014707472638971035 |
| }, |
| { |
| "index": 192, |
| "total": 252, |
| "projection": "layers.33.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3806, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.544459331194087, |
| "retained_delta_l2": 4.40694759884874, |
| "delta_retention_norm_ratio": 0.9697407937174312, |
| "delta_cosine": 0.795194596650398, |
| "relative_weight_rmse": 0.014216056005712982 |
| }, |
| { |
| "index": 193, |
| "total": 252, |
| "projection": "layers.33.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0305, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.1713470907351642, |
| "retained_delta_l2": 1.1620024134850435, |
| "delta_retention_norm_ratio": 0.9920222815901171, |
| "delta_cosine": 0.8277024355769212, |
| "relative_weight_rmse": 0.015548115562863228 |
| }, |
| { |
| "index": 194, |
| "total": 252, |
| "projection": "layers.33.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1278, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9580319646659914, |
| "retained_delta_l2": 1.810232136079443, |
| "delta_retention_norm_ratio": 0.9245161308631851, |
| "delta_cosine": 0.741492899620641, |
| "relative_weight_rmse": 0.012302995093158103 |
| }, |
| { |
| "index": 195, |
| "total": 252, |
| "projection": "layers.33.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1281, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.6943967329802574, |
| "retained_delta_l2": 1.562015056610022, |
| "delta_retention_norm_ratio": 0.9218709091008511, |
| "delta_cosine": 0.7426421898099466, |
| "relative_weight_rmse": 0.011918602777488662 |
| }, |
| { |
| "index": 196, |
| "total": 252, |
| "projection": "layers.33.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0293, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.2362498105050221, |
| "retained_delta_l2": 1.1869907840241474, |
| "delta_retention_norm_ratio": 0.9601544719665098, |
| "delta_cosine": 0.7867592705647769, |
| "relative_weight_rmse": 0.012739169630235064 |
| }, |
| { |
| "index": 197, |
| "total": 252, |
| "projection": "layers.34.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.42, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.248710622949176, |
| "retained_delta_l2": 3.016860809350351, |
| "delta_retention_norm_ratio": 0.9286332823979405, |
| "delta_cosine": 0.7449688814837238, |
| "relative_weight_rmse": 0.011825474077757182 |
| }, |
| { |
| "index": 198, |
| "total": 252, |
| "projection": "layers.34.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3793, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.5098548391117097, |
| "retained_delta_l2": 3.3057036996885336, |
| "delta_retention_norm_ratio": 0.9418348767167695, |
| "delta_cosine": 0.7568637240827227, |
| "relative_weight_rmse": 0.013100890845810342 |
| }, |
| { |
| "index": 199, |
| "total": 252, |
| "projection": "layers.34.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3798, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.119166573786764, |
| "retained_delta_l2": 2.7949884408142536, |
| "delta_retention_norm_ratio": 0.8960689898074445, |
| "delta_cosine": 0.7095439682266548, |
| "relative_weight_rmse": 0.011413755648549849 |
| }, |
| { |
| "index": 200, |
| "total": 252, |
| "projection": "layers.34.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0306, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.788059537523479, |
| "retained_delta_l2": 0.7264836013467776, |
| "delta_retention_norm_ratio": 0.9218638526091477, |
| "delta_cosine": 0.7523719001179419, |
| "relative_weight_rmse": 0.011623414101459404 |
| }, |
| { |
| "index": 201, |
| "total": 252, |
| "projection": "layers.34.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1286, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.366467424692969, |
| "retained_delta_l2": 2.295917305881346, |
| "delta_retention_norm_ratio": 0.970187580832313, |
| "delta_cosine": 0.7929682214572963, |
| "relative_weight_rmse": 0.014046097788924336 |
| }, |
| { |
| "index": 202, |
| "total": 252, |
| "projection": "layers.34.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1293, |
| "parameters": 16777216, |
| "reference_delta_l2": 2.790418420753438, |
| "retained_delta_l2": 2.803650672675589, |
| "delta_retention_norm_ratio": 1.0047420314544004, |
| "delta_cosine": 0.8597101316514167, |
| "relative_weight_rmse": 0.015277924377240636 |
| }, |
| { |
| "index": 203, |
| "total": 252, |
| "projection": "layers.34.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0293, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9825920197694892, |
| "retained_delta_l2": 0.8780767742150043, |
| "delta_retention_norm_ratio": 0.893633121934978, |
| "delta_cosine": 0.7155778670479893, |
| "relative_weight_rmse": 0.011416316401979395 |
| }, |
| { |
| "index": 204, |
| "total": 252, |
| "projection": "layers.35.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4207, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.331282571631798, |
| "retained_delta_l2": 2.0846773160858074, |
| "delta_retention_norm_ratio": 0.8942190626967292, |
| "delta_cosine": 0.71456817683431, |
| "relative_weight_rmse": 0.009809271791772265 |
| }, |
| { |
| "index": 205, |
| "total": 252, |
| "projection": "layers.35.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3807, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6185183399349397, |
| "retained_delta_l2": 2.2835312544709794, |
| "delta_retention_norm_ratio": 0.8720699869253986, |
| "delta_cosine": 0.6867799203989726, |
| "relative_weight_rmse": 0.010510595372961532 |
| }, |
| { |
| "index": 206, |
| "total": 252, |
| "projection": "layers.35.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3795, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6570990323321224, |
| "retained_delta_l2": 2.2838320956928437, |
| "delta_retention_norm_ratio": 0.8595208789370323, |
| "delta_cosine": 0.674113310648716, |
| "relative_weight_rmse": 0.010333697702349381 |
| }, |
| { |
| "index": 207, |
| "total": 252, |
| "projection": "layers.35.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0303, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.8324669053364727, |
| "retained_delta_l2": 0.7624225034713568, |
| "delta_retention_norm_ratio": 0.9158592354649763, |
| "delta_cosine": 0.7341526429180774, |
| "relative_weight_rmse": 0.012225562971949431 |
| }, |
| { |
| "index": 208, |
| "total": 252, |
| "projection": "layers.35.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1284, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9803735222413514, |
| "retained_delta_l2": 1.879211962319679, |
| "delta_retention_norm_ratio": 0.9489179395777926, |
| "delta_cosine": 0.7677733338888255, |
| "relative_weight_rmse": 0.012993498178701863 |
| }, |
| { |
| "index": 209, |
| "total": 252, |
| "projection": "layers.35.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1291, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.9183182575278923, |
| "retained_delta_l2": 1.8299753931083624, |
| "delta_retention_norm_ratio": 0.9539477539387151, |
| "delta_cosine": 0.7741720514765471, |
| "relative_weight_rmse": 0.013070551367355533 |
| }, |
| { |
| "index": 210, |
| "total": 252, |
| "projection": "layers.35.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0299, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.904757871002193, |
| "retained_delta_l2": 0.7982918615926153, |
| "delta_retention_norm_ratio": 0.8823265176000668, |
| "delta_cosine": 0.6993180987503474, |
| "relative_weight_rmse": 0.011424754057315271 |
| }, |
| { |
| "index": 211, |
| "total": 252, |
| "projection": "layers.4.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4206, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.2235173063590223, |
| "retained_delta_l2": 1.906815818828342, |
| "delta_retention_norm_ratio": 0.8575673386373257, |
| "delta_cosine": 0.6728611526585633, |
| "relative_weight_rmse": 0.01083163399891114 |
| }, |
| { |
| "index": 212, |
| "total": 252, |
| "projection": "layers.4.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3794, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.3121918812030633, |
| "retained_delta_l2": 2.921247503567911, |
| "delta_retention_norm_ratio": 0.8819680768334133, |
| "delta_cosine": 0.708916658869227, |
| "relative_weight_rmse": 0.010826649674697375 |
| }, |
| { |
| "index": 213, |
| "total": 252, |
| "projection": "layers.4.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3796, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.2985309176795696, |
| "retained_delta_l2": 3.176107062748976, |
| "delta_retention_norm_ratio": 0.9628853395690721, |
| "delta_cosine": 0.7950185998344161, |
| "relative_weight_rmse": 0.014017124847558887 |
| }, |
| { |
| "index": 214, |
| "total": 252, |
| "projection": "layers.4.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0311, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.5934111237094617, |
| "retained_delta_l2": 0.44817826231033947, |
| "delta_retention_norm_ratio": 0.7552576020293322, |
| "delta_cosine": 0.578349938854584, |
| "relative_weight_rmse": 0.009465869663785244 |
| }, |
| { |
| "index": 215, |
| "total": 252, |
| "projection": "layers.4.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1286, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.6304116923382745, |
| "retained_delta_l2": 1.4170472984587967, |
| "delta_retention_norm_ratio": 0.8691346517679356, |
| "delta_cosine": 0.6813502661109487, |
| "relative_weight_rmse": 0.011812966565665212 |
| }, |
| { |
| "index": 216, |
| "total": 252, |
| "projection": "layers.4.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.128, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.2617092660880085, |
| "retained_delta_l2": 0.9974311537855114, |
| "delta_retention_norm_ratio": 0.7905396120915366, |
| "delta_cosine": 0.6083472266105397, |
| "relative_weight_rmse": 0.010242073966784322 |
| }, |
| { |
| "index": 217, |
| "total": 252, |
| "projection": "layers.4.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0299, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.134908652300205, |
| "retained_delta_l2": 1.0772035945546896, |
| "delta_retention_norm_ratio": 0.9491544472512742, |
| "delta_cosine": 0.7690126096235654, |
| "relative_weight_rmse": 0.013531898852458491 |
| }, |
| { |
| "index": 218, |
| "total": 252, |
| "projection": "layers.5.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4206, |
| "parameters": 50331648, |
| "reference_delta_l2": 1.805244199081874, |
| "retained_delta_l2": 1.3438500178171813, |
| "delta_retention_norm_ratio": 0.7444145332252821, |
| "delta_cosine": 0.5696051761209536, |
| "relative_weight_rmse": 0.00904836797007844 |
| }, |
| { |
| "index": 219, |
| "total": 252, |
| "projection": "layers.5.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3799, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.9525778981332786, |
| "retained_delta_l2": 2.488721118044475, |
| "delta_retention_norm_ratio": 0.8428976995384034, |
| "delta_cosine": 0.6603585648895852, |
| "relative_weight_rmse": 0.009992629029317168 |
| }, |
| { |
| "index": 220, |
| "total": 252, |
| "projection": "layers.5.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3784, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.175459638209982, |
| "retained_delta_l2": 2.9611238043070127, |
| "delta_retention_norm_ratio": 0.9325024222245222, |
| "delta_cosine": 0.749286455071023, |
| "relative_weight_rmse": 0.013122280283399044 |
| }, |
| { |
| "index": 221, |
| "total": 252, |
| "projection": "layers.5.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0305, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.620912291932519, |
| "retained_delta_l2": 0.48920077989436106, |
| "delta_retention_norm_ratio": 0.7878742074372198, |
| "delta_cosine": 0.6085857604067588, |
| "relative_weight_rmse": 0.009867676262332552 |
| }, |
| { |
| "index": 222, |
| "total": 252, |
| "projection": "layers.5.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1294, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.4681154093186881, |
| "retained_delta_l2": 1.2332403890020849, |
| "delta_retention_norm_ratio": 0.8400159695717638, |
| "delta_cosine": 0.652071310389467, |
| "relative_weight_rmse": 0.01135209908927687 |
| }, |
| { |
| "index": 223, |
| "total": 252, |
| "projection": "layers.5.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1281, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.4695525761768655, |
| "retained_delta_l2": 1.2552732872923054, |
| "delta_retention_norm_ratio": 0.8541873952941369, |
| "delta_cosine": 0.6707500298719928, |
| "relative_weight_rmse": 0.011191405559489996 |
| }, |
| { |
| "index": 224, |
| "total": 252, |
| "projection": "layers.5.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0294, |
| "parameters": 4194304, |
| "reference_delta_l2": 1.0473823346102564, |
| "retained_delta_l2": 0.982970039394574, |
| "delta_retention_norm_ratio": 0.9385016406261509, |
| "delta_cosine": 0.7610612328634105, |
| "relative_weight_rmse": 0.012884923116135599 |
| }, |
| { |
| "index": 225, |
| "total": 252, |
| "projection": "layers.6.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4249, |
| "parameters": 50331648, |
| "reference_delta_l2": 4.764525398582769, |
| "retained_delta_l2": 4.777186666985024, |
| "delta_retention_norm_ratio": 1.0026574039055434, |
| "delta_cosine": 0.844118187407864, |
| "relative_weight_rmse": 0.015430922538590283 |
| }, |
| { |
| "index": 226, |
| "total": 252, |
| "projection": "layers.6.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3808, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.104598047780988, |
| "retained_delta_l2": 2.714107725700126, |
| "delta_retention_norm_ratio": 0.8742219391782571, |
| "delta_cosine": 0.6919393297464186, |
| "relative_weight_rmse": 0.010678733172134873 |
| }, |
| { |
| "index": 227, |
| "total": 252, |
| "projection": "layers.6.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3793, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.383879491960246, |
| "retained_delta_l2": 3.1954651440333475, |
| "delta_retention_norm_ratio": 0.9443200183769688, |
| "delta_cosine": 0.7629381293181445, |
| "relative_weight_rmse": 0.013213285321025748 |
| }, |
| { |
| "index": 228, |
| "total": 252, |
| "projection": "layers.6.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0309, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.5916733501140015, |
| "retained_delta_l2": 0.44635380713197037, |
| "delta_retention_norm_ratio": 0.7543922791958917, |
| "delta_cosine": 0.5777196202734631, |
| "relative_weight_rmse": 0.00956722997942124 |
| }, |
| { |
| "index": 229, |
| "total": 252, |
| "projection": "layers.6.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1293, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.3633300651272238, |
| "retained_delta_l2": 1.1047508640909813, |
| "delta_retention_norm_ratio": 0.8103326497005608, |
| "delta_cosine": 0.6262795360705317, |
| "relative_weight_rmse": 0.01038569498361775 |
| }, |
| { |
| "index": 230, |
| "total": 252, |
| "projection": "layers.6.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1277, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.4017194091687992, |
| "retained_delta_l2": 1.1622694495196952, |
| "delta_retention_norm_ratio": 0.8291741142465207, |
| "delta_cosine": 0.6443375660633781, |
| "relative_weight_rmse": 0.010715876944964214 |
| }, |
| { |
| "index": 231, |
| "total": 252, |
| "projection": "layers.6.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0293, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9850168241358457, |
| "retained_delta_l2": 0.909033533829316, |
| "delta_retention_norm_ratio": 0.9228609213114815, |
| "delta_cosine": 0.7392110575533334, |
| "relative_weight_rmse": 0.012391306263121522 |
| }, |
| { |
| "index": 232, |
| "total": 252, |
| "projection": "layers.7.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4244, |
| "parameters": 50331648, |
| "reference_delta_l2": 1.8440571621192021, |
| "retained_delta_l2": 1.3639695271647716, |
| "delta_retention_norm_ratio": 0.7396568583575194, |
| "delta_cosine": 0.5653240615098264, |
| "relative_weight_rmse": 0.00895583751832063 |
| }, |
| { |
| "index": 233, |
| "total": 252, |
| "projection": "layers.7.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.381, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.895113651182583, |
| "retained_delta_l2": 2.479818038399918, |
| "delta_retention_norm_ratio": 0.8565529154224993, |
| "delta_cosine": 0.673587761311191, |
| "relative_weight_rmse": 0.010347822361135678 |
| }, |
| { |
| "index": 234, |
| "total": 252, |
| "projection": "layers.7.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3808, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.977520666533661, |
| "retained_delta_l2": 2.694573987573266, |
| "delta_retention_norm_ratio": 0.9049723878861292, |
| "delta_cosine": 0.716699940461233, |
| "relative_weight_rmse": 0.012389541638697982 |
| }, |
| { |
| "index": 235, |
| "total": 252, |
| "projection": "layers.7.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0309, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.7219952103480194, |
| "retained_delta_l2": 0.6127900354038534, |
| "delta_retention_norm_ratio": 0.8487452916875633, |
| "delta_cosine": 0.6624336302250765, |
| "relative_weight_rmse": 0.010765056760676819 |
| }, |
| { |
| "index": 236, |
| "total": 252, |
| "projection": "layers.7.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1295, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.3651572381138637, |
| "retained_delta_l2": 1.1056655543698313, |
| "delta_retention_norm_ratio": 0.8099180984436981, |
| "delta_cosine": 0.6251010144527916, |
| "relative_weight_rmse": 0.010545079917670805 |
| }, |
| { |
| "index": 237, |
| "total": 252, |
| "projection": "layers.7.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1287, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.686328662577495, |
| "retained_delta_l2": 1.5244809371895633, |
| "delta_retention_norm_ratio": 0.9040236171159227, |
| "delta_cosine": 0.7180165440086638, |
| "relative_weight_rmse": 0.012119084769076126 |
| }, |
| { |
| "index": 238, |
| "total": 252, |
| "projection": "layers.7.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0297, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.9907258662029362, |
| "retained_delta_l2": 0.8943453671172424, |
| "delta_retention_norm_ratio": 0.9027172880273304, |
| "delta_cosine": 0.7180639043844059, |
| "relative_weight_rmse": 0.01199329350304583 |
| }, |
| { |
| "index": 239, |
| "total": 252, |
| "projection": "layers.8.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.4235, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6379165878454596, |
| "retained_delta_l2": 2.275639258967151, |
| "delta_retention_norm_ratio": 0.8626653585077146, |
| "delta_cosine": 0.6769355435295519, |
| "relative_weight_rmse": 0.01104824259294015 |
| }, |
| { |
| "index": 240, |
| "total": 252, |
| "projection": "layers.8.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3797, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.2034972953567924, |
| "retained_delta_l2": 1.660862525753068, |
| "delta_retention_norm_ratio": 0.7537393076237653, |
| "delta_cosine": 0.5771338803590647, |
| "relative_weight_rmse": 0.009469984393632263 |
| }, |
| { |
| "index": 241, |
| "total": 252, |
| "projection": "layers.8.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3806, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6781934239898906, |
| "retained_delta_l2": 2.282726225425444, |
| "delta_retention_norm_ratio": 0.8523380742324086, |
| "delta_cosine": 0.6656219044344226, |
| "relative_weight_rmse": 0.01142343276016386 |
| }, |
| { |
| "index": 242, |
| "total": 252, |
| "projection": "layers.8.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0306, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.8933313252195287, |
| "retained_delta_l2": 0.8099389001699583, |
| "delta_retention_norm_ratio": 0.9066500606266354, |
| "delta_cosine": 0.7198072328241356, |
| "relative_weight_rmse": 0.012128338473894976 |
| }, |
| { |
| "index": 243, |
| "total": 252, |
| "projection": "layers.8.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1308, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.569782748200427, |
| "retained_delta_l2": 1.3437943136159736, |
| "delta_retention_norm_ratio": 0.856038400954831, |
| "delta_cosine": 0.6700176988585014, |
| "relative_weight_rmse": 0.011293687479535023 |
| }, |
| { |
| "index": 244, |
| "total": 252, |
| "projection": "layers.8.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1285, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.593744600605696, |
| "retained_delta_l2": 1.3829338615481481, |
| "delta_retention_norm_ratio": 0.8677261469764791, |
| "delta_cosine": 0.6791018439784102, |
| "relative_weight_rmse": 0.011834264313765035 |
| }, |
| { |
| "index": 245, |
| "total": 252, |
| "projection": "layers.8.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0297, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.7964545648449372, |
| "retained_delta_l2": 0.6605401186244267, |
| "delta_retention_norm_ratio": 0.829350659510663, |
| "delta_cosine": 0.643459686260606, |
| "relative_weight_rmse": 0.010628014077483398 |
| }, |
| { |
| "index": 246, |
| "total": 252, |
| "projection": "layers.9.mlp.down_proj", |
| "shape": [ |
| 4096, |
| 12288 |
| ], |
| "seconds": 0.421, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.464680946104141, |
| "retained_delta_l2": 2.0932331294917286, |
| "delta_retention_norm_ratio": 0.8492917238640764, |
| "delta_cosine": 0.663738399009076, |
| "relative_weight_rmse": 0.01086888999748977 |
| }, |
| { |
| "index": 247, |
| "total": 252, |
| "projection": "layers.9.mlp.gate_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3797, |
| "parameters": 50331648, |
| "reference_delta_l2": 2.6928365402963883, |
| "retained_delta_l2": 2.2029761878075407, |
| "delta_retention_norm_ratio": 0.8180876019920129, |
| "delta_cosine": 0.6343326155503013, |
| "relative_weight_rmse": 0.010354534989285258 |
| }, |
| { |
| "index": 248, |
| "total": 252, |
| "projection": "layers.9.mlp.up_proj", |
| "shape": [ |
| 12288, |
| 4096 |
| ], |
| "seconds": 0.3799, |
| "parameters": 50331648, |
| "reference_delta_l2": 3.02142649667848, |
| "retained_delta_l2": 2.7392781767887966, |
| "delta_retention_norm_ratio": 0.9066175132177283, |
| "delta_cosine": 0.7215701331971082, |
| "relative_weight_rmse": 0.012375043511407021 |
| }, |
| { |
| "index": 249, |
| "total": 252, |
| "projection": "layers.9.self_attn.k_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0306, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.8387475883623314, |
| "retained_delta_l2": 0.7570215780053346, |
| "delta_retention_norm_ratio": 0.9025618535409822, |
| "delta_cosine": 0.7184386490493212, |
| "relative_weight_rmse": 0.0117868197567552 |
| }, |
| { |
| "index": 250, |
| "total": 252, |
| "projection": "layers.9.self_attn.o_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1284, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.5234736639526796, |
| "retained_delta_l2": 1.2978513350229988, |
| "delta_retention_norm_ratio": 0.851902704806659, |
| "delta_cosine": 0.6664206994831966, |
| "relative_weight_rmse": 0.01114751204441402 |
| }, |
| { |
| "index": 251, |
| "total": 252, |
| "projection": "layers.9.self_attn.q_proj", |
| "shape": [ |
| 4096, |
| 4096 |
| ], |
| "seconds": 0.1295, |
| "parameters": 16777216, |
| "reference_delta_l2": 1.80398629239049, |
| "retained_delta_l2": 1.6571476172590194, |
| "delta_retention_norm_ratio": 0.9186032201292991, |
| "delta_cosine": 0.734883791279542, |
| "relative_weight_rmse": 0.012697342392676805 |
| }, |
| { |
| "index": 252, |
| "total": 252, |
| "projection": "layers.9.self_attn.v_proj", |
| "shape": [ |
| 1024, |
| 4096 |
| ], |
| "seconds": 0.0299, |
| "parameters": 4194304, |
| "reference_delta_l2": 0.8011068966234224, |
| "retained_delta_l2": 0.6707916787744644, |
| "delta_retention_norm_ratio": 0.8373310498284032, |
| "delta_cosine": 0.6534620606103909, |
| "relative_weight_rmse": 0.010427694289242662 |
| } |
| ] |
| } |
|
|
