_ Git - rapid-gossip-sync-server/blob - src/serialization.rs

   1 use std::cmp::max;
   2 use std::collections::HashMap;
   3 use std::time::{SystemTime, UNIX_EPOCH};
   4
   5 use bitcoin::Network;
   6 use bitcoin::blockdata::constants::ChainHash;
   7 use lightning::ln::features::NodeFeatures;
   8 use lightning::ln::msgs::{UnsignedChannelAnnouncement, UnsignedChannelUpdate};
   9 use lightning::util::ser::{BigSize, Writeable};
  10 use crate::config;
  11
  12 use crate::lookup::{DeltaSet, DirectedUpdateDelta, NodeDeltaSet};
  13
  14 pub(super) struct SerializationSet {
  15         pub(super) announcements: Vec<UnsignedChannelAnnouncement>,
  16         pub(super) updates: Vec<UpdateSerialization>,
  17         pub(super) full_update_defaults: DefaultUpdateValues,
  18         pub(super) node_announcement_feature_defaults: Vec<NodeFeatures>,
  19         pub(super) node_mutations: NodeDeltaSet,
  20         pub(super) latest_seen: u32,
  21         pub(super) chain_hash: ChainHash,
  22 }
  23
  24 pub(super) struct DefaultUpdateValues {
  25         pub(super) cltv_expiry_delta: u16,
  26         pub(super) htlc_minimum_msat: u64,
  27         pub(super) fee_base_msat: u32,
  28         pub(super) fee_proportional_millionths: u32,
  29         pub(super) htlc_maximum_msat: u64,
  30 }
  31
  32 impl Default for DefaultUpdateValues {
  33         fn default() -> Self {
  34                 Self {
  35                         cltv_expiry_delta: 0,
  36                         htlc_minimum_msat: 0,
  37                         fee_base_msat: 0,
  38                         fee_proportional_millionths: 0,
  39                         htlc_maximum_msat: 0,
  40                 }
  41         }
  42 }
  43
  44 pub(super) struct MutatedProperties {
  45         pub(super) flags: bool,
  46         pub(super) cltv_expiry_delta: bool,
  47         pub(super) htlc_minimum_msat: bool,
  48         pub(super) fee_base_msat: bool,
  49         pub(super) fee_proportional_millionths: bool,
  50         pub(super) htlc_maximum_msat: bool,
  51 }
  52
  53 impl Default for MutatedProperties {
  54         fn default() -> Self {
  55                 Self {
  56                         flags: false,
  57                         cltv_expiry_delta: false,
  58                         htlc_minimum_msat: false,
  59                         fee_base_msat: false,
  60                         fee_proportional_millionths: false,
  61                         htlc_maximum_msat: false,
  62                 }
  63         }
  64 }
  65
  66 impl MutatedProperties {
  67         /// Does not include flags because the flag byte is always sent in full
  68         fn len(&self) -> u8 {
  69                 let mut mutations = 0;
  70                 if self.cltv_expiry_delta { mutations += 1; };
  71                 if self.htlc_minimum_msat { mutations += 1; };
  72                 if self.fee_base_msat { mutations += 1; };
  73                 if self.fee_proportional_millionths { mutations += 1; };
  74                 if self.htlc_maximum_msat { mutations += 1; };
  75                 mutations
  76         }
  77 }
  78
  79 pub(super) enum UpdateSerialization {
  80         Full(UnsignedChannelUpdate),
  81         Incremental(UnsignedChannelUpdate, MutatedProperties),
  82         Reminder(u64, u8),
  83 }
  84 impl UpdateSerialization {
  85         pub(super) fn scid(&self) -> u64 {
  86                 match self {
  87                         UpdateSerialization::Full(latest_update)|
  88                         UpdateSerialization::Incremental(latest_update, _) => latest_update.short_channel_id,
  89                         UpdateSerialization::Reminder(scid, _) => *scid,
  90                 }
  91         }
  92
  93         fn flags(&self) -> u8 {
  94                 match self {
  95                         UpdateSerialization::Full(latest_update)|
  96                         UpdateSerialization::Incremental(latest_update, _) => latest_update.flags,
  97                         UpdateSerialization::Reminder(_, flags) => *flags,
  98                 }
  99         }
 100 }
 101
 102 struct FullUpdateValueHistograms {
 103         cltv_expiry_delta: HashMap<u16, usize>,
 104         htlc_minimum_msat: HashMap<u64, usize>,
 105         fee_base_msat: HashMap<u32, usize>,
 106         fee_proportional_millionths: HashMap<u32, usize>,
 107         htlc_maximum_msat: HashMap<u64, usize>,
 108 }
 109
 110 pub(super) fn serialize_delta_set(channel_delta_set: DeltaSet, node_delta_set: NodeDeltaSet, last_sync_timestamp: u32) -> SerializationSet {
 111         let mut serialization_set = SerializationSet {
 112                 announcements: vec![],
 113                 updates: vec![],
 114                 full_update_defaults: Default::default(),
 115                 node_announcement_feature_defaults: vec![],
 116                 node_mutations: Default::default(),
 117                 chain_hash: ChainHash::using_genesis_block(Network::Bitcoin),
 118                 latest_seen: 0,
 119         };
 120
 121         let mut chain_hash_set = false;
 122
 123         let mut full_update_histograms = FullUpdateValueHistograms {
 124                 cltv_expiry_delta: Default::default(),
 125                 htlc_minimum_msat: Default::default(),
 126                 fee_base_msat: Default::default(),
 127                 fee_proportional_millionths: Default::default(),
 128                 htlc_maximum_msat: Default::default(),
 129         };
 130
 131         let mut record_full_update_in_histograms = |full_update: &UnsignedChannelUpdate| {
 132                 *full_update_histograms.cltv_expiry_delta.entry(full_update.cltv_expiry_delta).or_insert(0) += 1;
 133                 *full_update_histograms.htlc_minimum_msat.entry(full_update.htlc_minimum_msat).or_insert(0) += 1;
 134                 *full_update_histograms.fee_base_msat.entry(full_update.fee_base_msat).or_insert(0) += 1;
 135                 *full_update_histograms.fee_proportional_millionths.entry(full_update.fee_proportional_millionths).or_insert(0) += 1;
 136                 *full_update_histograms.htlc_maximum_msat.entry(full_update.htlc_maximum_msat).or_insert(0) += 1;
 137         };
 138
 139         // if the previous seen update happened more than 6 days ago, the client may have pruned it, and an incremental update wouldn't work
 140         let non_incremental_previous_update_threshold_timestamp = SystemTime::now().checked_sub(config::CHANNEL_REMINDER_AGE).unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs() as u32;
 141
 142         for (scid, channel_delta) in channel_delta_set.into_iter() {
 143
 144                 // any announcement chain hash is gonna be the same value. Just set it from the first one.
 145                 let channel_announcement_delta = channel_delta.announcement.as_ref().unwrap();
 146                 if !chain_hash_set {
 147                         chain_hash_set = true;
 148                         serialization_set.chain_hash = channel_announcement_delta.announcement.chain_hash;
 149                 }
 150
 151                 let current_announcement_seen = channel_announcement_delta.seen;
 152                 let is_new_announcement = current_announcement_seen >= last_sync_timestamp;
 153                 let is_newly_included_announcement = if let Some(first_update_seen) = channel_delta.first_bidirectional_updates_seen {
 154                         first_update_seen >= last_sync_timestamp
 155                 } else {
 156                         false
 157                 };
 158                 let send_announcement = is_new_announcement || is_newly_included_announcement;
 159                 if send_announcement {
 160                         serialization_set.latest_seen = max(serialization_set.latest_seen, current_announcement_seen);
 161                         serialization_set.announcements.push(channel_delta.announcement.unwrap().announcement);
 162                 }
 163
 164                 let direction_a_updates = channel_delta.updates.0;
 165                 let direction_b_updates = channel_delta.updates.1;
 166
 167                 let mut categorize_directed_update_serialization = |directed_updates: Option<DirectedUpdateDelta>| {
 168                         if let Some(updates) = directed_updates {
 169                                 if let Some(latest_update_delta) = updates.latest_update_after_seen {
 170                                         let latest_update = latest_update_delta.update;
 171                                         assert_eq!(latest_update.short_channel_id, scid, "Update in DB had wrong SCID column");
 172
 173                                         // the returned seen timestamp should be the latest of all the returned
 174                                         // announcements and latest updates
 175                                         serialization_set.latest_seen = max(serialization_set.latest_seen, latest_update_delta.seen);
 176
 177                                         if let Some(update_delta) = updates.last_update_before_seen {
 178                                                 let mutated_properties = updates.mutated_properties;
 179                                                 if send_announcement || mutated_properties.len() == 5 || update_delta.seen <= non_incremental_previous_update_threshold_timestamp {
 180                                                         // all five values have changed, it makes more sense to just
 181                                                         // serialize the update as a full update instead of as a change
 182                                                         // this way, the default values can be computed more efficiently
 183                                                         record_full_update_in_histograms(&latest_update);
 184                                                         serialization_set.updates.push(UpdateSerialization::Full(latest_update));
 185                                                 } else if mutated_properties.len() > 0 || mutated_properties.flags {
 186                                                         // we don't count flags as mutated properties
 187                                                         serialization_set.updates.push(
 188                                                                 UpdateSerialization::Incremental(latest_update, mutated_properties));
 189                                                 } else if channel_delta.requires_reminder {
 190                                                         if let Some(flags) = updates.serialization_update_flags {
 191                                                                 serialization_set.updates.push(UpdateSerialization::Reminder(scid, flags));
 192                                                         }
 193                                                 }
 194                                         } else {
 195                                                 // serialize the full update
 196                                                 record_full_update_in_histograms(&latest_update);
 197                                                 serialization_set.updates.push(UpdateSerialization::Full(latest_update));
 198                                         }
 199                                 } else if is_newly_included_announcement {
 200                                         if let Some(unannounced_update) = updates.last_update_before_seen {
 201                                                 serialization_set.updates.push(UpdateSerialization::Full(unannounced_update.update));
 202                                         }
 203                                 } else if let Some(flags) = updates.serialization_update_flags {
 204                                         serialization_set.updates.push(UpdateSerialization::Reminder(scid, flags));
 205                                 }
 206                         }
 207                 };
 208
 209                 categorize_directed_update_serialization(direction_a_updates);
 210                 categorize_directed_update_serialization(direction_b_updates);
 211         }
 212
 213         let default_update_values = DefaultUpdateValues {
 214                 cltv_expiry_delta: find_most_common_histogram_entry_with_default(full_update_histograms.cltv_expiry_delta, 0),
 215                 htlc_minimum_msat: find_most_common_histogram_entry_with_default(full_update_histograms.htlc_minimum_msat, 0),
 216                 fee_base_msat: find_most_common_histogram_entry_with_default(full_update_histograms.fee_base_msat, 0),
 217                 fee_proportional_millionths: find_most_common_histogram_entry_with_default(full_update_histograms.fee_proportional_millionths, 0),
 218                 htlc_maximum_msat: find_most_common_histogram_entry_with_default(full_update_histograms.htlc_maximum_msat, 0),
 219         };
 220
 221         serialization_set.full_update_defaults = default_update_values;
 222
 223         serialization_set.node_mutations = node_delta_set.into_iter().filter(|(_id, delta)| {
 224                 // either something changed, or this node is new
 225                 delta.has_feature_set_changed || delta.has_address_set_changed || delta.last_details_before_seen.is_none()
 226         }).collect();
 227
 228         let mut node_feature_histogram: HashMap<&NodeFeatures, usize> = Default::default();
 229         for (_id, delta) in serialization_set.node_mutations.iter() {
 230                 if delta.has_feature_set_changed || delta.last_details_before_seen.is_none() {
 231                         if let Some(latest_details) = delta.latest_details_after_seen.as_ref() {
 232                                 *node_feature_histogram.entry(&latest_details.features).or_insert(0) += 1;
 233                         };
 234                 }
 235         }
 236         serialization_set.node_announcement_feature_defaults = find_leading_histogram_entries(node_feature_histogram, config::NODE_DEFAULT_FEATURE_COUNT as usize);
 237
 238         serialization_set
 239 }
 240
 241 pub fn serialize_stripped_channel_announcement(announcement: &UnsignedChannelAnnouncement, node_id_a_index: usize, node_id_b_index: usize, previous_scid: u64) -> Vec<u8> {
 242         let mut stripped_announcement = vec![];
 243
 244         announcement.features.write(&mut stripped_announcement).unwrap();
 245
 246         if previous_scid > announcement.short_channel_id {
 247                 panic!("unsorted scids!");
 248         }
 249         let scid_delta = BigSize(announcement.short_channel_id - previous_scid);
 250         scid_delta.write(&mut stripped_announcement).unwrap();
 251
 252         // write indices of node ids rather than the node IDs themselves
 253         BigSize(node_id_a_index as u64).write(&mut stripped_announcement).unwrap();
 254         BigSize(node_id_b_index as u64).write(&mut stripped_announcement).unwrap();
 255
 256         // println!("serialized CA: {}, \n{:?}\n{:?}\n", announcement.short_channel_id, announcement.node_id_1, announcement.node_id_2);
 257         stripped_announcement
 258 }
 259
 260 pub(super) fn serialize_stripped_channel_update(update: &UpdateSerialization, default_values: &DefaultUpdateValues, previous_scid: u64) -> Vec<u8> {
 261         let mut serialized_flags = update.flags();
 262
 263         if previous_scid > update.scid() {
 264                 panic!("unsorted scids!");
 265         }
 266
 267         let mut delta_serialization = Vec::new();
 268         let mut prefixed_serialization = Vec::new();
 269
 270         match update {
 271                 UpdateSerialization::Full(latest_update) => {
 272                         if latest_update.cltv_expiry_delta != default_values.cltv_expiry_delta {
 273                                 serialized_flags |= 0b_0100_0000;
 274                                 latest_update.cltv_expiry_delta.write(&mut delta_serialization).unwrap();
 275                         }
 276
 277                         if latest_update.htlc_minimum_msat != default_values.htlc_minimum_msat {
 278                                 serialized_flags |= 0b_0010_0000;
 279                                 latest_update.htlc_minimum_msat.write(&mut delta_serialization).unwrap();
 280                         }
 281
 282                         if latest_update.fee_base_msat != default_values.fee_base_msat {
 283                                 serialized_flags |= 0b_0001_0000;
 284                                 latest_update.fee_base_msat.write(&mut delta_serialization).unwrap();
 285                         }
 286
 287                         if latest_update.fee_proportional_millionths != default_values.fee_proportional_millionths {
 288                                 serialized_flags |= 0b_0000_1000;
 289                                 latest_update.fee_proportional_millionths.write(&mut delta_serialization).unwrap();
 290                         }
 291
 292                         if latest_update.htlc_maximum_msat != default_values.htlc_maximum_msat {
 293                                 serialized_flags |= 0b_0000_0100;
 294                                 latest_update.htlc_maximum_msat.write(&mut delta_serialization).unwrap();
 295                         }
 296                 }
 297                 UpdateSerialization::Incremental(latest_update, mutated_properties) => {
 298                         // indicate that this update is incremental
 299                         serialized_flags |= 0b_1000_0000;
 300
 301                         if mutated_properties.cltv_expiry_delta {
 302                                 serialized_flags |= 0b_0100_0000;
 303                                 latest_update.cltv_expiry_delta.write(&mut delta_serialization).unwrap();
 304                         }
 305
 306                         if mutated_properties.htlc_minimum_msat {
 307                                 serialized_flags |= 0b_0010_0000;
 308                                 latest_update.htlc_minimum_msat.write(&mut delta_serialization).unwrap();
 309                         }
 310
 311                         if mutated_properties.fee_base_msat {
 312                                 serialized_flags |= 0b_0001_0000;
 313                                 latest_update.fee_base_msat.write(&mut delta_serialization).unwrap();
 314                         }
 315
 316                         if mutated_properties.fee_proportional_millionths {
 317                                 serialized_flags |= 0b_0000_1000;
 318                                 latest_update.fee_proportional_millionths.write(&mut delta_serialization).unwrap();
 319                         }
 320
 321                         if mutated_properties.htlc_maximum_msat {
 322                                 serialized_flags |= 0b_0000_0100;
 323                                 latest_update.htlc_maximum_msat.write(&mut delta_serialization).unwrap();
 324                         }
 325                 },
 326                 UpdateSerialization::Reminder(_, _) => {
 327                         // indicate that this update is incremental
 328                         serialized_flags |= 0b_1000_0000;
 329                 }
 330         }
 331         let scid_delta = BigSize(update.scid() - previous_scid);
 332         scid_delta.write(&mut prefixed_serialization).unwrap();
 333
 334         serialized_flags.write(&mut prefixed_serialization).unwrap();
 335         prefixed_serialization.append(&mut delta_serialization);
 336
 337         prefixed_serialization
 338 }
 339
 340 pub(super) fn find_most_common_histogram_entry_with_default<T: Copy>(histogram: HashMap<T, usize>, default: T) -> T {
 341         let most_frequent_entry = histogram.iter().max_by(|a, b| a.1.cmp(&b.1));
 342         if let Some(entry_details) = most_frequent_entry {
 343                 // .0 is the value
 344                 // .1 is the frequency
 345                 return entry_details.0.to_owned();
 346         }
 347         // the default should pretty much always be a 0 as T
 348         // though for htlc maximum msat it could be a u64::max
 349         default
 350 }
 351
 352 pub(super) fn find_leading_histogram_entries(histogram: HashMap<&NodeFeatures, usize>, count: usize) -> Vec<NodeFeatures> {
 353         let mut entry_counts: Vec<_> = histogram.iter().filter(|&(_, &count)| count > 1).collect();
 354         entry_counts.sort_by(|a, b| b.1.cmp(&a.1));
 355         entry_counts.into_iter().take(count).map(|(&features, _count)| features.clone()).collect()
 356 }