Skip to content

Commit bb159bb

Browse files
thekidthekid
committed
Fix fragment encoding when streaming
1 parent 8085296 commit bb159bb

File tree

1 file changed

+85
-72
lines changed

1 file changed

+85
-72
lines changed

‎src/main/php/io/streams/compress/SnappyOutputStream.class.php

Lines changed: 85 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -51,70 +51,87 @@ private function equals32(int $a, int $b): bool {
5151
/** Compresses a fragment and returns last emitted position */
5252
private function fragment() {
5353
$end= min(strlen($this->buffer), Snappy::BLOCK_SIZE);
54-
if ($end <= Snappy::INPUT_MARGIN) return 0;
55-
5654
$pos= $emit= 0;
57-
$bits= 1;
58-
while ((1 << $bits) <= $end && $bits <= Snappy::HASH_BITS) {
59-
$bits++;
60-
}
61-
$bits--;
62-
$shift= 32 - $bits;
63-
$hashtable= array_fill(0, 1 << $bits, 0);
64-
65-
$start= $pos;
66-
$limit= $end - Snappy::INPUT_MARGIN;
67-
$next= ((unpack('V', $this->buffer, ++$pos)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
68-
69-
// Emit literals
70-
next: $forward= $pos;
71-
$skip= 32;
72-
do {
73-
$pos= $forward;
74-
$hash= $next;
75-
$forward+= ($skip & 0xffffffff) >> 5;
76-
$skip++;
77-
if ($pos > $limit) return $emit;
78-
79-
$next= ((unpack('V', $this->buffer, $forward)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
80-
$candidate= $start + $hashtable[$hash];
81-
$hashtable[$hash]= ($pos - $start) & 0xffff;
82-
} while (!$this->equals32($pos, $candidate));
83-
84-
$this->out->write($this->literal($pos - $emit).substr($this->buffer, $emit, $pos - $emit));
85-
86-
// Emit copy instructions
87-
do {
88-
$offset= $pos - $candidate;
89-
$matched= 4;
90-
while ($pos + $matched < $end && $this->buffer[$pos + $matched] === $this->buffer[$candidate + $matched]) {
91-
$matched++;
92-
}
93-
$pos+= $matched;
55+
$out= '';
9456

95-
while ($matched >= 68) {
96-
$this->out->write($this->copy($offset, 64));
97-
$matched-= 64;
98-
}
99-
if ($matched > 64) {
100-
$this->out->write($this->copy($offset, 60));
101-
$matched-= 60;
102-
}
103-
$this->out->write($this->copy($offset, $matched));
104-
$emit= $pos;
57+
// Compare 4-byte offsets in data at offsets a and b
58+
$equals32= fn($a, $b) => (
59+
$this->buffer[$a] === $this->buffer[$b] &&
60+
$this->buffer[$a + 1] === $this->buffer[$b + 1] &&
61+
$this->buffer[$a + 2] === $this->buffer[$b + 2] &&
62+
$this->buffer[$a + 3] === $this->buffer[$b + 3]
63+
);
10564

106-
if ($pos >= $limit) return $emit;
65+
if ($end >= Snappy::INPUT_MARGIN) {
66+
$bits= 1;
67+
while ((1 << $bits) <= $end && $bits <= Snappy::HASH_BITS) {
68+
$bits++;
69+
}
70+
$bits--;
71+
$shift= 32 - $bits;
72+
$hashtable= array_fill(0, 1 << $bits, 0);
73+
74+
$start= $pos;
75+
$limit= $end - Snappy::INPUT_MARGIN;
76+
$next= ((unpack('V', $this->buffer, ++$pos)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
77+
78+
// Emit literals
79+
next: $forward= $pos;
80+
$skip= 32;
81+
do {
82+
$pos= $forward;
83+
$hash= $next;
84+
$forward+= ($skip & 0xffffffff) >> 5;
85+
$skip++;
86+
if ($pos > $limit || $forward > $limit) goto emit;
87+
88+
$next= ((unpack('V', $this->buffer, $forward)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
89+
$candidate= $start + $hashtable[$hash];
90+
$hashtable[$hash]= ($pos - $start) & 0xffff;
91+
} while (!$equals32($pos, $candidate));
92+
93+
$out.= $this->literal($pos - $emit).substr($this->buffer, $emit, $pos - $emit);
94+
95+
// Emit copy instructions
96+
do {
97+
$offset= $pos - $candidate;
98+
$matched= 4;
99+
while ($pos + $matched < $end && $this->buffer[$pos + $matched] === $this->buffer[$candidate + $matched]) {
100+
$matched++;
101+
}
102+
$pos+= $matched;
103+
104+
while ($matched >= 68) {
105+
$out.= $this->copy($offset, 64);
106+
$matched-= 64;
107+
}
108+
if ($matched > 64) {
109+
$out.= $this->copy($offset, 60);
110+
$matched-= 60;
111+
}
112+
$out.= $this->copy($offset, $matched);
113+
$emit= $pos;
114+
115+
if ($pos >= $limit) goto emit;
116+
117+
$hash= ((unpack('V', $this->buffer, $pos - 1)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
118+
$hashtable[$hash]= ($pos - 1 - $start) & 0xffff;
119+
$hash= ((unpack('V', $this->buffer, $pos)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
120+
$candidate= $start + $hashtable[$hash];
121+
$hashtable[$hash]= ($pos - $start) & 0xffff;
122+
} while ($equals32($pos, $candidate));
123+
124+
$pos++;
125+
$next= ((unpack('V', $this->buffer, $pos)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
126+
goto next;
127+
}
107128

108-
$hash= ((unpack('V', $this->buffer, $pos - 1)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
109-
$hashtable[$hash]= ($pos - 1 - $start) & 0xffff;
110-
$hash= ((unpack('V', $this->buffer, $pos)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
111-
$candidate= $start + $hashtable[$hash];
112-
$hashtable[$hash]= ($pos - $start) & 0xffff;
113-
} while ($this->equals32($pos, $candidate));
129+
emit: if ($emit < $end) {
130+
$out.= $this->literal($end - $emit).substr($this->buffer, $emit, $end - $emit);
131+
}
114132

115-
$pos++;
116-
$next= ((unpack('V', $this->buffer, $pos)[1] * Snappy::HASH_KEY) & 0xffffffff) >> $shift;
117-
goto next;
133+
$this->buffer= substr($this->buffer, $end);
134+
return $out;
118135
}
119136

120137
/**
@@ -124,20 +141,21 @@ private function fragment() {
124141
* @return void
125142
*/
126143
public function write($arg) {
127-
if (strlen($this->buffer) <= Snappy::BLOCK_SIZE) {
128-
$this->buffer.= $arg;
129-
} else {
130-
$this->buffer= substr($this->buffer, $this->fragment());
144+
$this->buffer.= $arg;
145+
if (strlen($this->buffer) > Snappy::BLOCK_SIZE) {
146+
$this->out->write($this->fragment());
131147
}
132148
}
133149

134150
/**
135-
* Flush this buffer (except if it's smaller than the input margin)
151+
* Flush this buffer
136152
*
137153
* @return void
138154
*/
139155
public function flush() {
140-
$this->buffer= substr($this->buffer, $this->fragment());
156+
if (strlen($this->buffer) > 0) {
157+
$this->out->write($this->fragment());
158+
}
141159
}
142160

143161
/**
@@ -148,15 +166,10 @@ public function flush() {
148166
* @return void
149167
*/
150168
public function close() {
151-
$end= strlen($this->buffer);
152-
if ($end > 0) {
153-
$emit= $this->fragment();
154-
if ($emit < $end) {
155-
$this->out->write($this->literal($end - $emit).substr($this->buffer, $emit, $end - $emit));
156-
}
169+
if (strlen($this->buffer) > 0) {
170+
$this->out->write($this->fragment());
157171
$this->buffer= '';
158172
}
159-
160173
$this->out->close();
161174
}
162175
}

0 commit comments

Comments
 (0)