]> git.bitcoin.ninja Git - rapid-gossip-sync-server/blob - src/serialization.rs
Redo indexes for new SELECTs and trivially tweak one old query
[rapid-gossip-sync-server] / src / serialization.rs
1 use std::cmp::max;
2 use std::collections::HashMap;
3
4 use bitcoin::BlockHash;
5 use bitcoin::hashes::Hash;
6 use lightning::ln::msgs::{UnsignedChannelAnnouncement, UnsignedChannelUpdate};
7 use lightning::util::ser::{BigSize, Writeable};
8
9 use crate::lookup::{DeltaSet, DirectedUpdateDelta};
10
11 pub(super) struct SerializationSet {
12         pub(super) announcements: Vec<UnsignedChannelAnnouncement>,
13         pub(super) updates: Vec<UpdateSerialization>,
14         pub(super) full_update_defaults: DefaultUpdateValues,
15         pub(super) latest_seen: u32,
16         pub(super) chain_hash: BlockHash,
17 }
18
19 pub(super) struct DefaultUpdateValues {
20         pub(super) cltv_expiry_delta: u16,
21         pub(super) htlc_minimum_msat: u64,
22         pub(super) fee_base_msat: u32,
23         pub(super) fee_proportional_millionths: u32,
24         pub(super) htlc_maximum_msat: u64,
25 }
26
27 impl Default for DefaultUpdateValues {
28         fn default() -> Self {
29                 Self {
30                         cltv_expiry_delta: 0,
31                         htlc_minimum_msat: 0,
32                         fee_base_msat: 0,
33                         fee_proportional_millionths: 0,
34                         htlc_maximum_msat: 0,
35                 }
36         }
37 }
38
39 pub(super) struct MutatedProperties {
40         pub(super) flags: bool,
41         pub(super) cltv_expiry_delta: bool,
42         pub(super) htlc_minimum_msat: bool,
43         pub(super) fee_base_msat: bool,
44         pub(super) fee_proportional_millionths: bool,
45         pub(super) htlc_maximum_msat: bool,
46 }
47
48 impl Default for MutatedProperties {
49         fn default() -> Self {
50                 Self {
51                         flags: false,
52                         cltv_expiry_delta: false,
53                         htlc_minimum_msat: false,
54                         fee_base_msat: false,
55                         fee_proportional_millionths: false,
56                         htlc_maximum_msat: false,
57                 }
58         }
59 }
60
61 impl MutatedProperties {
62         /// Does not include flags because the flag byte is always sent in full
63         fn len(&self) -> u8 {
64                 let mut mutations = 0;
65                 if self.cltv_expiry_delta { mutations += 1; };
66                 if self.htlc_minimum_msat { mutations += 1; };
67                 if self.fee_base_msat { mutations += 1; };
68                 if self.fee_proportional_millionths { mutations += 1; };
69                 if self.htlc_maximum_msat { mutations += 1; };
70                 mutations
71         }
72 }
73
74 pub(super) enum UpdateSerialization {
75         Full(UnsignedChannelUpdate),
76         Incremental(UnsignedChannelUpdate, MutatedProperties),
77         Reminder(u64, u8),
78 }
79 impl UpdateSerialization {
80         pub(super) fn scid(&self) -> u64 {
81                 match self {
82                         UpdateSerialization::Full(latest_update)|
83                         UpdateSerialization::Incremental(latest_update, _) => latest_update.short_channel_id,
84                         UpdateSerialization::Reminder(scid, _) => *scid,
85                 }
86         }
87
88         fn flags(&self) -> u8 {
89                 match self {
90                         UpdateSerialization::Full(latest_update)|
91                         UpdateSerialization::Incremental(latest_update, _) => latest_update.flags,
92                         UpdateSerialization::Reminder(_, flags) => *flags,
93                 }
94         }
95 }
96
97 struct FullUpdateValueHistograms {
98         cltv_expiry_delta: HashMap<u16, usize>,
99         htlc_minimum_msat: HashMap<u64, usize>,
100         fee_base_msat: HashMap<u32, usize>,
101         fee_proportional_millionths: HashMap<u32, usize>,
102         htlc_maximum_msat: HashMap<u64, usize>,
103 }
104
105 pub(super) fn serialize_delta_set(delta_set: DeltaSet, last_sync_timestamp: u32) -> SerializationSet {
106         let mut serialization_set = SerializationSet {
107                 announcements: vec![],
108                 updates: vec![],
109                 full_update_defaults: Default::default(),
110                 chain_hash: BlockHash::all_zeros(),
111                 latest_seen: 0,
112         };
113
114         let mut chain_hash_set = false;
115
116         let mut full_update_histograms = FullUpdateValueHistograms {
117                 cltv_expiry_delta: Default::default(),
118                 htlc_minimum_msat: Default::default(),
119                 fee_base_msat: Default::default(),
120                 fee_proportional_millionths: Default::default(),
121                 htlc_maximum_msat: Default::default(),
122         };
123
124         let mut record_full_update_in_histograms = |full_update: &UnsignedChannelUpdate| {
125                 *full_update_histograms.cltv_expiry_delta.entry(full_update.cltv_expiry_delta).or_insert(0) += 1;
126                 *full_update_histograms.htlc_minimum_msat.entry(full_update.htlc_minimum_msat).or_insert(0) += 1;
127                 *full_update_histograms.fee_base_msat.entry(full_update.fee_base_msat).or_insert(0) += 1;
128                 *full_update_histograms.fee_proportional_millionths.entry(full_update.fee_proportional_millionths).or_insert(0) += 1;
129                 *full_update_histograms.htlc_maximum_msat.entry(full_update.htlc_maximum_msat).or_insert(0) += 1;
130         };
131
132         for (scid, channel_delta) in delta_set.into_iter() {
133
134                 // any announcement chain hash is gonna be the same value. Just set it from the first one.
135                 let channel_announcement_delta = channel_delta.announcement.as_ref().unwrap();
136                 if !chain_hash_set {
137                         chain_hash_set = true;
138                         serialization_set.chain_hash = channel_announcement_delta.announcement.chain_hash.clone();
139                 }
140
141                 let current_announcement_seen = channel_announcement_delta.seen;
142                 let is_new_announcement = current_announcement_seen >= last_sync_timestamp;
143                 let is_newly_updated_announcement = if let Some(first_update_seen) = channel_delta.first_bidirectional_updates_seen {
144                         first_update_seen >= last_sync_timestamp
145                 } else {
146                         false
147                 };
148                 let send_announcement = is_new_announcement || is_newly_updated_announcement;
149                 if send_announcement {
150                         serialization_set.latest_seen = max(serialization_set.latest_seen, current_announcement_seen);
151                         serialization_set.announcements.push(channel_delta.announcement.unwrap().announcement);
152                 }
153
154                 let direction_a_updates = channel_delta.updates.0;
155                 let direction_b_updates = channel_delta.updates.1;
156
157                 let mut categorize_directed_update_serialization = |directed_updates: Option<DirectedUpdateDelta>| {
158                         if let Some(updates) = directed_updates {
159                                 if let Some(latest_update_delta) = updates.latest_update_after_seen {
160                                         let latest_update = latest_update_delta.update;
161                                         assert_eq!(latest_update.short_channel_id, scid, "Update in DB had wrong SCID column");
162
163                                         // the returned seen timestamp should be the latest of all the returned
164                                         // announcements and latest updates
165                                         serialization_set.latest_seen = max(serialization_set.latest_seen, latest_update_delta.seen);
166
167                                         if updates.last_update_before_seen.is_some() {
168                                                 let mutated_properties = updates.mutated_properties;
169                                                 if mutated_properties.len() == 5 {
170                                                         // all five values have changed, it makes more sense to just
171                                                         // serialize the update as a full update instead of as a change
172                                                         // this way, the default values can be computed more efficiently
173                                                         record_full_update_in_histograms(&latest_update);
174                                                         serialization_set.updates.push(UpdateSerialization::Full(latest_update));
175                                                 } else if mutated_properties.len() > 0 || mutated_properties.flags {
176                                                         // we don't count flags as mutated properties
177                                                         serialization_set.updates.push(
178                                                                 UpdateSerialization::Incremental(latest_update, mutated_properties));
179                                                 }
180                                         } else {
181                                                 // serialize the full update
182                                                 record_full_update_in_histograms(&latest_update);
183                                                 serialization_set.updates.push(UpdateSerialization::Full(latest_update));
184                                         }
185                                 } else if let Some(flags) = updates.serialization_update_flags {
186                                         serialization_set.updates.push(UpdateSerialization::Reminder(scid, flags));
187                                 }
188                         }
189                 };
190
191                 categorize_directed_update_serialization(direction_a_updates);
192                 categorize_directed_update_serialization(direction_b_updates);
193         }
194
195         let default_update_values = DefaultUpdateValues {
196                 cltv_expiry_delta: find_most_common_histogram_entry_with_default(full_update_histograms.cltv_expiry_delta, 0),
197                 htlc_minimum_msat: find_most_common_histogram_entry_with_default(full_update_histograms.htlc_minimum_msat, 0),
198                 fee_base_msat: find_most_common_histogram_entry_with_default(full_update_histograms.fee_base_msat, 0),
199                 fee_proportional_millionths: find_most_common_histogram_entry_with_default(full_update_histograms.fee_proportional_millionths, 0),
200                 htlc_maximum_msat: find_most_common_histogram_entry_with_default(full_update_histograms.htlc_maximum_msat, 0),
201         };
202
203         serialization_set.full_update_defaults = default_update_values;
204         serialization_set
205 }
206
207 pub fn serialize_stripped_channel_announcement(announcement: &UnsignedChannelAnnouncement, node_id_a_index: usize, node_id_b_index: usize, previous_scid: u64) -> Vec<u8> {
208         let mut stripped_announcement = vec![];
209
210         announcement.features.write(&mut stripped_announcement).unwrap();
211
212         if previous_scid > announcement.short_channel_id {
213                 panic!("unsorted scids!");
214         }
215         let scid_delta = BigSize(announcement.short_channel_id - previous_scid);
216         scid_delta.write(&mut stripped_announcement).unwrap();
217
218         // write indices of node ids rather than the node IDs themselves
219         BigSize(node_id_a_index as u64).write(&mut stripped_announcement).unwrap();
220         BigSize(node_id_b_index as u64).write(&mut stripped_announcement).unwrap();
221
222         // println!("serialized CA: {}, \n{:?}\n{:?}\n", announcement.short_channel_id, announcement.node_id_1, announcement.node_id_2);
223         stripped_announcement
224 }
225
226 pub(super) fn serialize_stripped_channel_update(update: &UpdateSerialization, default_values: &DefaultUpdateValues, previous_scid: u64) -> Vec<u8> {
227         let mut serialized_flags = update.flags();
228
229         if previous_scid > update.scid() {
230                 panic!("unsorted scids!");
231         }
232
233         let mut delta_serialization = Vec::new();
234         let mut prefixed_serialization = Vec::new();
235
236         match update {
237                 UpdateSerialization::Full(latest_update) => {
238                         if latest_update.cltv_expiry_delta != default_values.cltv_expiry_delta {
239                                 serialized_flags |= 0b_0100_0000;
240                                 latest_update.cltv_expiry_delta.write(&mut delta_serialization).unwrap();
241                         }
242
243                         if latest_update.htlc_minimum_msat != default_values.htlc_minimum_msat {
244                                 serialized_flags |= 0b_0010_0000;
245                                 latest_update.htlc_minimum_msat.write(&mut delta_serialization).unwrap();
246                         }
247
248                         if latest_update.fee_base_msat != default_values.fee_base_msat {
249                                 serialized_flags |= 0b_0001_0000;
250                                 latest_update.fee_base_msat.write(&mut delta_serialization).unwrap();
251                         }
252
253                         if latest_update.fee_proportional_millionths != default_values.fee_proportional_millionths {
254                                 serialized_flags |= 0b_0000_1000;
255                                 latest_update.fee_proportional_millionths.write(&mut delta_serialization).unwrap();
256                         }
257
258                         if latest_update.htlc_maximum_msat != default_values.htlc_maximum_msat {
259                                 serialized_flags |= 0b_0000_0100;
260                                 latest_update.htlc_maximum_msat.write(&mut delta_serialization).unwrap();
261                         }
262                 }
263                 UpdateSerialization::Incremental(latest_update, mutated_properties) => {
264                         // indicate that this update is incremental
265                         serialized_flags |= 0b_1000_0000;
266
267                         if mutated_properties.cltv_expiry_delta {
268                                 serialized_flags |= 0b_0100_0000;
269                                 latest_update.cltv_expiry_delta.write(&mut delta_serialization).unwrap();
270                         }
271
272                         if mutated_properties.htlc_minimum_msat {
273                                 serialized_flags |= 0b_0010_0000;
274                                 latest_update.htlc_minimum_msat.write(&mut delta_serialization).unwrap();
275                         }
276
277                         if mutated_properties.fee_base_msat {
278                                 serialized_flags |= 0b_0001_0000;
279                                 latest_update.fee_base_msat.write(&mut delta_serialization).unwrap();
280                         }
281
282                         if mutated_properties.fee_proportional_millionths {
283                                 serialized_flags |= 0b_0000_1000;
284                                 latest_update.fee_proportional_millionths.write(&mut delta_serialization).unwrap();
285                         }
286
287                         if mutated_properties.htlc_maximum_msat {
288                                 serialized_flags |= 0b_0000_0100;
289                                 latest_update.htlc_maximum_msat.write(&mut delta_serialization).unwrap();
290                         }
291                 },
292                 UpdateSerialization::Reminder(_, _) => {
293                         // indicate that this update is incremental
294                         serialized_flags |= 0b_1000_0000;
295                 }
296         }
297         let scid_delta = BigSize(update.scid() - previous_scid);
298         scid_delta.write(&mut prefixed_serialization).unwrap();
299
300         serialized_flags.write(&mut prefixed_serialization).unwrap();
301         prefixed_serialization.append(&mut delta_serialization);
302
303         prefixed_serialization
304 }
305
306 pub(super) fn find_most_common_histogram_entry_with_default<T: Copy>(histogram: HashMap<T, usize>, default: T) -> T {
307         let most_frequent_entry = histogram.iter().max_by(|a, b| a.1.cmp(&b.1));
308         if let Some(entry_details) = most_frequent_entry {
309                 // .0 is the value
310                 // .1 is the frequency
311                 return entry_details.0.to_owned();
312         }
313         // the default should pretty much always be a 0 as T
314         // though for htlc maximum msat it could be a u64::max
315         default
316 }