_ Git - rapid-gossip-sync-server/blob - src/serialization.rs

   1 use std::cmp::max;
   2 use std::collections::HashMap;
   3
   4 use bitcoin::BlockHash;
   5 use lightning::ln::msgs::{UnsignedChannelAnnouncement, UnsignedChannelUpdate};
   6 use lightning::util::ser::{BigSize, Writeable};
   7
   8 use crate::lookup::{DeltaSet, DirectedUpdateDelta};
   9
  10 pub(super) struct SerializationSet {
  11         pub(super) announcements: Vec<UnsignedChannelAnnouncement>,
  12         pub(super) updates: Vec<UpdateSerialization>,
  13         pub(super) full_update_defaults: DefaultUpdateValues,
  14         pub(super) latest_seen: u32,
  15         pub(super) chain_hash: BlockHash,
  16 }
  17
  18 pub(super) struct DefaultUpdateValues {
  19         pub(super) cltv_expiry_delta: u16,
  20         pub(super) htlc_minimum_msat: u64,
  21         pub(super) fee_base_msat: u32,
  22         pub(super) fee_proportional_millionths: u32,
  23         pub(super) htlc_maximum_msat: u64,
  24 }
  25
  26 impl Default for DefaultUpdateValues {
  27         fn default() -> Self {
  28                 Self {
  29                         cltv_expiry_delta: 0,
  30                         htlc_minimum_msat: 0,
  31                         fee_base_msat: 0,
  32                         fee_proportional_millionths: 0,
  33                         htlc_maximum_msat: 0,
  34                 }
  35         }
  36 }
  37
  38 pub(super) struct MutatedProperties {
  39         pub(super) flags: bool,
  40         pub(super) cltv_expiry_delta: bool,
  41         pub(super) htlc_minimum_msat: bool,
  42         pub(super) fee_base_msat: bool,
  43         pub(super) fee_proportional_millionths: bool,
  44         pub(super) htlc_maximum_msat: bool,
  45 }
  46
  47 impl Default for MutatedProperties {
  48         fn default() -> Self {
  49                 Self {
  50                         flags: false,
  51                         cltv_expiry_delta: false,
  52                         htlc_minimum_msat: false,
  53                         fee_base_msat: false,
  54                         fee_proportional_millionths: false,
  55                         htlc_maximum_msat: false,
  56                 }
  57         }
  58 }
  59
  60 pub(super) struct UpdateSerialization {
  61         pub(super) update: UnsignedChannelUpdate,
  62         pub(super) mechanism: UpdateSerializationMechanism,
  63 }
  64
  65 impl MutatedProperties {
  66         /// Does not include flags because the flag byte is always sent in full
  67         fn len(&self) -> u8 {
  68                 let mut mutations = 0;
  69                 if self.cltv_expiry_delta { mutations += 1; };
  70                 if self.htlc_minimum_msat { mutations += 1; };
  71                 if self.fee_base_msat { mutations += 1; };
  72                 if self.fee_proportional_millionths { mutations += 1; };
  73                 if self.htlc_maximum_msat { mutations += 1; };
  74                 mutations
  75         }
  76 }
  77
  78 pub(super) enum UpdateSerializationMechanism {
  79         Full,
  80         Incremental(MutatedProperties),
  81 }
  82
  83 struct FullUpdateValueHistograms {
  84         cltv_expiry_delta: HashMap<u16, usize>,
  85         htlc_minimum_msat: HashMap<u64, usize>,
  86         fee_base_msat: HashMap<u32, usize>,
  87         fee_proportional_millionths: HashMap<u32, usize>,
  88         htlc_maximum_msat: HashMap<u64, usize>,
  89 }
  90
  91 pub(super) fn serialize_delta_set(delta_set: DeltaSet, last_sync_timestamp: u32) -> SerializationSet {
  92         let mut serialization_set = SerializationSet {
  93                 announcements: vec![],
  94                 updates: vec![],
  95                 full_update_defaults: Default::default(),
  96                 chain_hash: Default::default(),
  97                 latest_seen: 0,
  98         };
  99
 100         let mut chain_hash_set = false;
 101
 102         let mut full_update_histograms = FullUpdateValueHistograms {
 103                 cltv_expiry_delta: Default::default(),
 104                 htlc_minimum_msat: Default::default(),
 105                 fee_base_msat: Default::default(),
 106                 fee_proportional_millionths: Default::default(),
 107                 htlc_maximum_msat: Default::default(),
 108         };
 109
 110         let mut record_full_update_in_histograms = |full_update: &UnsignedChannelUpdate| {
 111                 *full_update_histograms.cltv_expiry_delta.entry(full_update.cltv_expiry_delta).or_insert(0) += 1;
 112                 *full_update_histograms.htlc_minimum_msat.entry(full_update.htlc_minimum_msat).or_insert(0) += 1;
 113                 *full_update_histograms.fee_base_msat.entry(full_update.fee_base_msat).or_insert(0) += 1;
 114                 *full_update_histograms.fee_proportional_millionths.entry(full_update.fee_proportional_millionths).or_insert(0) += 1;
 115                 *full_update_histograms.htlc_maximum_msat.entry(full_update.htlc_maximum_msat).or_insert(0) += 1;
 116         };
 117
 118         for (_scid, channel_delta) in delta_set.into_iter() {
 119
 120                 // any announcement chain hash is gonna be the same value. Just set it from the first one.
 121                 let channel_announcement_delta = channel_delta.announcement.as_ref().unwrap();
 122                 if !chain_hash_set {
 123                         chain_hash_set = true;
 124                         serialization_set.chain_hash = channel_announcement_delta.announcement.chain_hash.clone();
 125                 }
 126
 127                 let current_announcement_seen = channel_announcement_delta.seen;
 128                 let is_new_announcement = current_announcement_seen >= last_sync_timestamp;
 129                 let is_newly_updated_announcement = if let Some(first_update_seen) = channel_delta.first_update_seen {
 130                         first_update_seen >= last_sync_timestamp
 131                 } else {
 132                         false
 133                 };
 134                 let send_announcement = is_new_announcement || is_newly_updated_announcement;
 135                 if send_announcement {
 136                         serialization_set.latest_seen = max(serialization_set.latest_seen, current_announcement_seen);
 137                         serialization_set.announcements.push(channel_delta.announcement.unwrap().announcement);
 138                 }
 139
 140                 let direction_a_updates = channel_delta.updates.0;
 141                 let direction_b_updates = channel_delta.updates.1;
 142
 143                 let mut categorize_directed_update_serialization = |directed_updates: Option<DirectedUpdateDelta>| {
 144                         if let Some(updates) = directed_updates {
 145                                 if let Some(latest_update_delta) = updates.latest_update_after_seen {
 146                                         let latest_update = latest_update_delta.update;
 147
 148                                         // the returned seen timestamp should be the latest of all the returned
 149                                         // announcements and latest updates
 150                                         serialization_set.latest_seen = max(serialization_set.latest_seen, latest_update_delta.seen);
 151
 152                                         if updates.last_update_before_seen.is_some() {
 153                                                 let mutated_properties = updates.mutated_properties;
 154                                                 if mutated_properties.len() == 5 {
 155                                                         // all five values have changed, it makes more sense to just
 156                                                         // serialize the update as a full update instead of as a change
 157                                                         // this way, the default values can be computed more efficiently
 158                                                         record_full_update_in_histograms(&latest_update);
 159                                                         serialization_set.updates.push(UpdateSerialization {
 160                                                                 update: latest_update,
 161                                                                 mechanism: UpdateSerializationMechanism::Full,
 162                                                         });
 163                                                 } else if mutated_properties.len() > 0 || mutated_properties.flags {
 164                                                         // we don't count flags as mutated properties
 165                                                         serialization_set.updates.push(UpdateSerialization {
 166                                                                 update: latest_update,
 167                                                                 mechanism: UpdateSerializationMechanism::Incremental(mutated_properties),
 168                                                         });
 169                                                 }
 170                                         } else {
 171                                                 // serialize the full update
 172                                                 record_full_update_in_histograms(&latest_update);
 173                                                 serialization_set.updates.push(UpdateSerialization {
 174                                                         update: latest_update,
 175                                                         mechanism: UpdateSerializationMechanism::Full,
 176                                                 });
 177                                         }
 178                                 }
 179                         };
 180                 };
 181
 182                 categorize_directed_update_serialization(direction_a_updates);
 183                 categorize_directed_update_serialization(direction_b_updates);
 184         }
 185
 186         let default_update_values = DefaultUpdateValues {
 187                 cltv_expiry_delta: find_most_common_histogram_entry_with_default(full_update_histograms.cltv_expiry_delta, 0),
 188                 htlc_minimum_msat: find_most_common_histogram_entry_with_default(full_update_histograms.htlc_minimum_msat, 0),
 189                 fee_base_msat: find_most_common_histogram_entry_with_default(full_update_histograms.fee_base_msat, 0),
 190                 fee_proportional_millionths: find_most_common_histogram_entry_with_default(full_update_histograms.fee_proportional_millionths, 0),
 191                 htlc_maximum_msat: find_most_common_histogram_entry_with_default(full_update_histograms.htlc_maximum_msat, 0),
 192         };
 193
 194         serialization_set.full_update_defaults = default_update_values;
 195         serialization_set
 196 }
 197
 198 pub fn serialize_stripped_channel_announcement(announcement: &UnsignedChannelAnnouncement, node_id_a_index: usize, node_id_b_index: usize, previous_scid: u64) -> Vec<u8> {
 199         let mut stripped_announcement = vec![];
 200
 201         announcement.features.write(&mut stripped_announcement).unwrap();
 202
 203         if previous_scid > announcement.short_channel_id {
 204                 panic!("unsorted scids!");
 205         }
 206         let scid_delta = BigSize(announcement.short_channel_id - previous_scid);
 207         scid_delta.write(&mut stripped_announcement).unwrap();
 208
 209         // write indices of node ids rather than the node IDs themselves
 210         BigSize(node_id_a_index as u64).write(&mut stripped_announcement).unwrap();
 211         BigSize(node_id_b_index as u64).write(&mut stripped_announcement).unwrap();
 212
 213         // println!("serialized CA: {}, \n{:?}\n{:?}\n", announcement.short_channel_id, announcement.node_id_1, announcement.node_id_2);
 214         stripped_announcement
 215 }
 216
 217 pub(super) fn serialize_stripped_channel_update(update: &UpdateSerialization, default_values: &DefaultUpdateValues, previous_scid: u64) -> Vec<u8> {
 218         let latest_update = &update.update;
 219         let mut serialized_flags = latest_update.flags;
 220
 221         if previous_scid > latest_update.short_channel_id {
 222                 panic!("unsorted scids!");
 223         }
 224
 225         let mut delta_serialization = Vec::new();
 226         let mut prefixed_serialization = Vec::new();
 227
 228         match &update.mechanism {
 229                 UpdateSerializationMechanism::Full => {
 230                         if latest_update.cltv_expiry_delta != default_values.cltv_expiry_delta {
 231                                 serialized_flags |= 0b_0100_0000;
 232                                 latest_update.cltv_expiry_delta.write(&mut delta_serialization).unwrap();
 233                         }
 234
 235                         if latest_update.htlc_minimum_msat != default_values.htlc_minimum_msat {
 236                                 serialized_flags |= 0b_0010_0000;
 237                                 latest_update.htlc_minimum_msat.write(&mut delta_serialization).unwrap();
 238                         }
 239
 240                         if latest_update.fee_base_msat != default_values.fee_base_msat {
 241                                 serialized_flags |= 0b_0001_0000;
 242                                 latest_update.fee_base_msat.write(&mut delta_serialization).unwrap();
 243                         }
 244
 245                         if latest_update.fee_proportional_millionths != default_values.fee_proportional_millionths {
 246                                 serialized_flags |= 0b_0000_1000;
 247                                 latest_update.fee_proportional_millionths.write(&mut delta_serialization).unwrap();
 248                         }
 249
 250                         if latest_update.htlc_maximum_msat != default_values.htlc_maximum_msat {
 251                                 serialized_flags |= 0b_0000_0100;
 252                                 latest_update.htlc_maximum_msat.write(&mut delta_serialization).unwrap();
 253                         }
 254                 }
 255
 256                 UpdateSerializationMechanism::Incremental(mutated_properties) => {
 257                         // indicate that this update is incremental
 258                         serialized_flags |= 0b_1000_0000;
 259
 260                         if mutated_properties.cltv_expiry_delta {
 261                                 serialized_flags |= 0b_0100_0000;
 262                                 latest_update.cltv_expiry_delta.write(&mut delta_serialization).unwrap();
 263                         }
 264
 265                         if mutated_properties.htlc_minimum_msat {
 266                                 serialized_flags |= 0b_0010_0000;
 267                                 latest_update.htlc_minimum_msat.write(&mut delta_serialization).unwrap();
 268                         }
 269
 270                         if mutated_properties.fee_base_msat {
 271                                 serialized_flags |= 0b_0001_0000;
 272                                 latest_update.fee_base_msat.write(&mut delta_serialization).unwrap();
 273                         }
 274
 275                         if mutated_properties.fee_proportional_millionths {
 276                                 serialized_flags |= 0b_0000_1000;
 277                                 latest_update.fee_proportional_millionths.write(&mut delta_serialization).unwrap();
 278                         }
 279
 280                         if mutated_properties.htlc_maximum_msat {
 281                                 serialized_flags |= 0b_0000_0100;
 282                                 latest_update.htlc_maximum_msat.write(&mut delta_serialization).unwrap();
 283                         }
 284                 }
 285         }
 286         let scid_delta = BigSize(latest_update.short_channel_id - previous_scid);
 287         scid_delta.write(&mut prefixed_serialization).unwrap();
 288
 289         serialized_flags.write(&mut prefixed_serialization).unwrap();
 290         prefixed_serialization.append(&mut delta_serialization);
 291
 292         prefixed_serialization
 293 }
 294
 295 pub(super) fn find_most_common_histogram_entry_with_default<T: Copy>(histogram: HashMap<T, usize>, default: T) -> T {
 296         let most_frequent_entry = histogram.iter().max_by(|a, b| a.1.cmp(&b.1));
 297         if let Some(entry_details) = most_frequent_entry {
 298                 // .0 is the value
 299                 // .1 is the frequency
 300                 return entry_details.0.to_owned();
 301         }
 302         // the default should pretty much always be a 0 as T
 303         // though for htlc maximum msat it could be a u64::max
 304         default
 305 }