_ Git - rapid-gossip-sync-server/blob - src/serialization.rs

   1 use std::cmp::max;
   2 use std::collections::HashMap;
   3
   4 use bitcoin::BlockHash;
   5 use bitcoin::hashes::Hash;
   6 use lightning::ln::msgs::{UnsignedChannelAnnouncement, UnsignedChannelUpdate};
   7 use lightning::util::ser::{BigSize, Writeable};
   8
   9 use crate::lookup::{DeltaSet, DirectedUpdateDelta};
  10
  11 pub(super) struct SerializationSet {
  12         pub(super) announcements: Vec<UnsignedChannelAnnouncement>,
  13         pub(super) updates: Vec<UpdateSerialization>,
  14         pub(super) full_update_defaults: DefaultUpdateValues,
  15         pub(super) latest_seen: u32,
  16         pub(super) chain_hash: BlockHash,
  17 }
  18
  19 pub(super) struct DefaultUpdateValues {
  20         pub(super) cltv_expiry_delta: u16,
  21         pub(super) htlc_minimum_msat: u64,
  22         pub(super) fee_base_msat: u32,
  23         pub(super) fee_proportional_millionths: u32,
  24         pub(super) htlc_maximum_msat: u64,
  25 }
  26
  27 impl Default for DefaultUpdateValues {
  28         fn default() -> Self {
  29                 Self {
  30                         cltv_expiry_delta: 0,
  31                         htlc_minimum_msat: 0,
  32                         fee_base_msat: 0,
  33                         fee_proportional_millionths: 0,
  34                         htlc_maximum_msat: 0,
  35                 }
  36         }
  37 }
  38
  39 pub(super) struct MutatedProperties {
  40         pub(super) flags: bool,
  41         pub(super) cltv_expiry_delta: bool,
  42         pub(super) htlc_minimum_msat: bool,
  43         pub(super) fee_base_msat: bool,
  44         pub(super) fee_proportional_millionths: bool,
  45         pub(super) htlc_maximum_msat: bool,
  46 }
  47
  48 impl Default for MutatedProperties {
  49         fn default() -> Self {
  50                 Self {
  51                         flags: false,
  52                         cltv_expiry_delta: false,
  53                         htlc_minimum_msat: false,
  54                         fee_base_msat: false,
  55                         fee_proportional_millionths: false,
  56                         htlc_maximum_msat: false,
  57                 }
  58         }
  59 }
  60
  61 impl MutatedProperties {
  62         /// Does not include flags because the flag byte is always sent in full
  63         fn len(&self) -> u8 {
  64                 let mut mutations = 0;
  65                 if self.cltv_expiry_delta { mutations += 1; };
  66                 if self.htlc_minimum_msat { mutations += 1; };
  67                 if self.fee_base_msat { mutations += 1; };
  68                 if self.fee_proportional_millionths { mutations += 1; };
  69                 if self.htlc_maximum_msat { mutations += 1; };
  70                 mutations
  71         }
  72 }
  73
  74 pub(super) enum UpdateSerialization {
  75         Full(UnsignedChannelUpdate),
  76         Incremental(UnsignedChannelUpdate, MutatedProperties),
  77         Reminder(u64, u8),
  78 }
  79 impl UpdateSerialization {
  80         pub(super) fn scid(&self) -> u64 {
  81                 match self {
  82                         UpdateSerialization::Full(latest_update)|
  83                         UpdateSerialization::Incremental(latest_update, _) => latest_update.short_channel_id,
  84                         UpdateSerialization::Reminder(scid, _) => *scid,
  85                 }
  86         }
  87
  88         fn flags(&self) -> u8 {
  89                 match self {
  90                         UpdateSerialization::Full(latest_update)|
  91                         UpdateSerialization::Incremental(latest_update, _) => latest_update.flags,
  92                         UpdateSerialization::Reminder(_, flags) => *flags,
  93                 }
  94         }
  95 }
  96
  97 struct FullUpdateValueHistograms {
  98         cltv_expiry_delta: HashMap<u16, usize>,
  99         htlc_minimum_msat: HashMap<u64, usize>,
 100         fee_base_msat: HashMap<u32, usize>,
 101         fee_proportional_millionths: HashMap<u32, usize>,
 102         htlc_maximum_msat: HashMap<u64, usize>,
 103 }
 104
 105 pub(super) fn serialize_delta_set(delta_set: DeltaSet, last_sync_timestamp: u32) -> SerializationSet {
 106         let mut serialization_set = SerializationSet {
 107                 announcements: vec![],
 108                 updates: vec![],
 109                 full_update_defaults: Default::default(),
 110                 chain_hash: BlockHash::all_zeros(),
 111                 latest_seen: 0,
 112         };
 113
 114         let mut chain_hash_set = false;
 115
 116         let mut full_update_histograms = FullUpdateValueHistograms {
 117                 cltv_expiry_delta: Default::default(),
 118                 htlc_minimum_msat: Default::default(),
 119                 fee_base_msat: Default::default(),
 120                 fee_proportional_millionths: Default::default(),
 121                 htlc_maximum_msat: Default::default(),
 122         };
 123
 124         let mut record_full_update_in_histograms = |full_update: &UnsignedChannelUpdate| {
 125                 *full_update_histograms.cltv_expiry_delta.entry(full_update.cltv_expiry_delta).or_insert(0) += 1;
 126                 *full_update_histograms.htlc_minimum_msat.entry(full_update.htlc_minimum_msat).or_insert(0) += 1;
 127                 *full_update_histograms.fee_base_msat.entry(full_update.fee_base_msat).or_insert(0) += 1;
 128                 *full_update_histograms.fee_proportional_millionths.entry(full_update.fee_proportional_millionths).or_insert(0) += 1;
 129                 *full_update_histograms.htlc_maximum_msat.entry(full_update.htlc_maximum_msat).or_insert(0) += 1;
 130         };
 131
 132         for (scid, channel_delta) in delta_set.into_iter() {
 133
 134                 // any announcement chain hash is gonna be the same value. Just set it from the first one.
 135                 let channel_announcement_delta = channel_delta.announcement.as_ref().unwrap();
 136                 if !chain_hash_set {
 137                         chain_hash_set = true;
 138                         serialization_set.chain_hash = channel_announcement_delta.announcement.chain_hash.clone();
 139                 }
 140
 141                 let current_announcement_seen = channel_announcement_delta.seen;
 142                 let is_new_announcement = current_announcement_seen >= last_sync_timestamp;
 143                 let is_newly_updated_announcement = if let Some(first_update_seen) = channel_delta.first_bidirectional_updates_seen {
 144                         first_update_seen >= last_sync_timestamp
 145                 } else {
 146                         false
 147                 };
 148                 let send_announcement = is_new_announcement || is_newly_updated_announcement;
 149                 if send_announcement {
 150                         serialization_set.latest_seen = max(serialization_set.latest_seen, current_announcement_seen);
 151                         serialization_set.announcements.push(channel_delta.announcement.unwrap().announcement);
 152                 }
 153
 154                 let direction_a_updates = channel_delta.updates.0;
 155                 let direction_b_updates = channel_delta.updates.1;
 156
 157                 let mut categorize_directed_update_serialization = |directed_updates: Option<DirectedUpdateDelta>| {
 158                         if let Some(updates) = directed_updates {
 159                                 if let Some(latest_update_delta) = updates.latest_update_after_seen {
 160                                         let latest_update = latest_update_delta.update;
 161                                         assert_eq!(latest_update.short_channel_id, scid, "Update in DB had wrong SCID column");
 162
 163                                         // the returned seen timestamp should be the latest of all the returned
 164                                         // announcements and latest updates
 165                                         serialization_set.latest_seen = max(serialization_set.latest_seen, latest_update_delta.seen);
 166
 167                                         if updates.last_update_before_seen.is_some() {
 168                                                 let mutated_properties = updates.mutated_properties;
 169                                                 if mutated_properties.len() == 5 {
 170                                                         // all five values have changed, it makes more sense to just
 171                                                         // serialize the update as a full update instead of as a change
 172                                                         // this way, the default values can be computed more efficiently
 173                                                         record_full_update_in_histograms(&latest_update);
 174                                                         serialization_set.updates.push(UpdateSerialization::Full(latest_update));
 175                                                 } else if mutated_properties.len() > 0 || mutated_properties.flags {
 176                                                         // we don't count flags as mutated properties
 177                                                         serialization_set.updates.push(
 178                                                                 UpdateSerialization::Incremental(latest_update, mutated_properties));
 179                                                 }
 180                                         } else {
 181                                                 // serialize the full update
 182                                                 record_full_update_in_histograms(&latest_update);
 183                                                 serialization_set.updates.push(UpdateSerialization::Full(latest_update));
 184                                         }
 185                                 } else if let Some(flags) = updates.serialization_update_flags {
 186                                         serialization_set.updates.push(UpdateSerialization::Reminder(scid, flags));
 187                                 }
 188                         }
 189                 };
 190
 191                 categorize_directed_update_serialization(direction_a_updates);
 192                 categorize_directed_update_serialization(direction_b_updates);
 193         }
 194
 195         let default_update_values = DefaultUpdateValues {
 196                 cltv_expiry_delta: find_most_common_histogram_entry_with_default(full_update_histograms.cltv_expiry_delta, 0),
 197                 htlc_minimum_msat: find_most_common_histogram_entry_with_default(full_update_histograms.htlc_minimum_msat, 0),
 198                 fee_base_msat: find_most_common_histogram_entry_with_default(full_update_histograms.fee_base_msat, 0),
 199                 fee_proportional_millionths: find_most_common_histogram_entry_with_default(full_update_histograms.fee_proportional_millionths, 0),
 200                 htlc_maximum_msat: find_most_common_histogram_entry_with_default(full_update_histograms.htlc_maximum_msat, 0),
 201         };
 202
 203         serialization_set.full_update_defaults = default_update_values;
 204         serialization_set
 205 }
 206
 207 pub fn serialize_stripped_channel_announcement(announcement: &UnsignedChannelAnnouncement, node_id_a_index: usize, node_id_b_index: usize, previous_scid: u64) -> Vec<u8> {
 208         let mut stripped_announcement = vec![];
 209
 210         announcement.features.write(&mut stripped_announcement).unwrap();
 211
 212         if previous_scid > announcement.short_channel_id {
 213                 panic!("unsorted scids!");
 214         }
 215         let scid_delta = BigSize(announcement.short_channel_id - previous_scid);
 216         scid_delta.write(&mut stripped_announcement).unwrap();
 217
 218         // write indices of node ids rather than the node IDs themselves
 219         BigSize(node_id_a_index as u64).write(&mut stripped_announcement).unwrap();
 220         BigSize(node_id_b_index as u64).write(&mut stripped_announcement).unwrap();
 221
 222         // println!("serialized CA: {}, \n{:?}\n{:?}\n", announcement.short_channel_id, announcement.node_id_1, announcement.node_id_2);
 223         stripped_announcement
 224 }
 225
 226 pub(super) fn serialize_stripped_channel_update(update: &UpdateSerialization, default_values: &DefaultUpdateValues, previous_scid: u64) -> Vec<u8> {
 227         let mut serialized_flags = update.flags();
 228
 229         if previous_scid > update.scid() {
 230                 panic!("unsorted scids!");
 231         }
 232
 233         let mut delta_serialization = Vec::new();
 234         let mut prefixed_serialization = Vec::new();
 235
 236         match update {
 237                 UpdateSerialization::Full(latest_update) => {
 238                         if latest_update.cltv_expiry_delta != default_values.cltv_expiry_delta {
 239                                 serialized_flags |= 0b_0100_0000;
 240                                 latest_update.cltv_expiry_delta.write(&mut delta_serialization).unwrap();
 241                         }
 242
 243                         if latest_update.htlc_minimum_msat != default_values.htlc_minimum_msat {
 244                                 serialized_flags |= 0b_0010_0000;
 245                                 latest_update.htlc_minimum_msat.write(&mut delta_serialization).unwrap();
 246                         }
 247
 248                         if latest_update.fee_base_msat != default_values.fee_base_msat {
 249                                 serialized_flags |= 0b_0001_0000;
 250                                 latest_update.fee_base_msat.write(&mut delta_serialization).unwrap();
 251                         }
 252
 253                         if latest_update.fee_proportional_millionths != default_values.fee_proportional_millionths {
 254                                 serialized_flags |= 0b_0000_1000;
 255                                 latest_update.fee_proportional_millionths.write(&mut delta_serialization).unwrap();
 256                         }
 257
 258                         if latest_update.htlc_maximum_msat != default_values.htlc_maximum_msat {
 259                                 serialized_flags |= 0b_0000_0100;
 260                                 latest_update.htlc_maximum_msat.write(&mut delta_serialization).unwrap();
 261                         }
 262                 }
 263                 UpdateSerialization::Incremental(latest_update, mutated_properties) => {
 264                         // indicate that this update is incremental
 265                         serialized_flags |= 0b_1000_0000;
 266
 267                         if mutated_properties.cltv_expiry_delta {
 268                                 serialized_flags |= 0b_0100_0000;
 269                                 latest_update.cltv_expiry_delta.write(&mut delta_serialization).unwrap();
 270                         }
 271
 272                         if mutated_properties.htlc_minimum_msat {
 273                                 serialized_flags |= 0b_0010_0000;
 274                                 latest_update.htlc_minimum_msat.write(&mut delta_serialization).unwrap();
 275                         }
 276
 277                         if mutated_properties.fee_base_msat {
 278                                 serialized_flags |= 0b_0001_0000;
 279                                 latest_update.fee_base_msat.write(&mut delta_serialization).unwrap();
 280                         }
 281
 282                         if mutated_properties.fee_proportional_millionths {
 283                                 serialized_flags |= 0b_0000_1000;
 284                                 latest_update.fee_proportional_millionths.write(&mut delta_serialization).unwrap();
 285                         }
 286
 287                         if mutated_properties.htlc_maximum_msat {
 288                                 serialized_flags |= 0b_0000_0100;
 289                                 latest_update.htlc_maximum_msat.write(&mut delta_serialization).unwrap();
 290                         }
 291                 },
 292                 UpdateSerialization::Reminder(_, _) => {
 293                         // indicate that this update is incremental
 294                         serialized_flags |= 0b_1000_0000;
 295                 }
 296         }
 297         let scid_delta = BigSize(update.scid() - previous_scid);
 298         scid_delta.write(&mut prefixed_serialization).unwrap();
 299
 300         serialized_flags.write(&mut prefixed_serialization).unwrap();
 301         prefixed_serialization.append(&mut delta_serialization);
 302
 303         prefixed_serialization
 304 }
 305
 306 pub(super) fn find_most_common_histogram_entry_with_default<T: Copy>(histogram: HashMap<T, usize>, default: T) -> T {
 307         let most_frequent_entry = histogram.iter().max_by(|a, b| a.1.cmp(&b.1));
 308         if let Some(entry_details) = most_frequent_entry {
 309                 // .0 is the value
 310                 // .1 is the frequency
 311                 return entry_details.0.to_owned();
 312         }
 313         // the default should pretty much always be a 0 as T
 314         // though for htlc maximum msat it could be a u64::max
 315         default
 316 }