folly::RecordIO — append-only 로그 파일 포맷

1
+----------------+----------------+----------------+----------------+
2
|  magic (4)     |  length (4)    |   fileId (4)   |  checksum (4)  |
3
+----------------+----------------+----------------+----------------+
4
|                          payload (length bytes)                   |
5
+-------------------------------------------------------------------+
6

7
총 헤더: 16 bytes
8
magic    : 0xFADE0001 (sync word)
9
length   : payload 길이 (big endian)
10
fileId   : 같은 파일을 식별하는 32-bit (random per file)
11
checksum : SpookyHashV2 결과 32-bit (header + payload)

1
#include <folly/io/RecordIO.h>
2

3
// write
4
{
5
  folly::File f("log.recordio", O_WRONLY | O_CREAT | O_APPEND);
6
  folly::RecordIOWriter writer(std::move(f));
7

8
  writer.write(folly::IOBuf::wrapBuffer(data1, len1));
9
  writer.write(folly::IOBuf::wrapBuffer(data2, len2));
10
}
11

12
// read
13
{
14
  folly::File f("log.recordio", O_RDONLY);
15
  folly::RecordIOReader reader(std::move(f));
16

17
  for (auto rec : reader) {
18
    folly::ByteRange payload = rec.first;
19
    off_t            offset  = rec.second;
20
    Process(payload);
21
  }
22
}

1
folly::RecordIOReader reader(std::move(f));
2

3
// 파일 중간 임의 offset에서 다음 valid record 찾기
4
auto it = reader.seek(offset);
5
for (; it != reader.end(); ++it) {
6
  Process(it->first);
7
}

1
// folly/io/RecordIO.cpp 약식
2
struct Header {
3
  uint32_t magic;
4
  uint32_t length;
5
  uint32_t fileId;
6
  uint32_t checksum;
7
};
8

9
void RecordIOWriter::write(std::unique_ptr<folly::IOBuf> buf) {
10
  Header h;
11
  h.magic    = kMagic;
12
  h.length   = buf->computeChainDataLength();
13
  h.fileId   = fileId_;
14
  h.checksum = computeChecksum(h, *buf);   // header + payload
15

16
  pwrite(fd_, &h, sizeof(h), offset_);
17
  pwriteIOBuf(fd_, *buf, offset_ + sizeof(h));
18
  offset_ += sizeof(h) + h.length;
19
}
20

21
static uint32_t computeChecksum(const Header& h, const IOBuf& buf) {
22
  folly::hash::SpookyHashV2 sp;
23
  sp.Init(0, 0);
24
  // header without checksum field
25
  Header partial = h;
26
  partial.checksum = 0;
27
  sp.Update(&partial, sizeof(partial));
28
  // payload
29
  for (auto& range : buf) {
30
    sp.Update(range.data(), range.size());
31
  }
32
  uint64_t h1, h2;
33
  sp.Final(&h1, &h2);
34
  return static_cast<uint32_t>(h1);
35
}

1
// reader iterator
2
auto RecordIOReader::Iterator::operator++() -> Iterator& {
3
  for (;;) {
4
    if (offset_ >= fileSize_) { /* end */ return *this; }
5
    Header h;
6
    pread(fd_, &h, sizeof(h), offset_);
7
    if (h.magic != kMagic) {
8
      offset_ = findNextMagic(offset_ + 1);   // forward search
9
      continue;
10
    }
11
    if (h.fileId != fileId_) {
12
      offset_ = findNextMagic(offset_ + 1);
13
      continue;
14
    }
15
    // 길이 sanity
16
    if (offset_ + sizeof(h) + h.length > fileSize_) {
17
      offset_ = findNextMagic(offset_ + 1);
18
      continue;
19
    }
20
    auto buf = readPayload(offset_ + sizeof(h), h.length);
21
    if (computeChecksum(h, *buf) != h.checksum) {
22
      offset_ = findNextMagic(offset_ + 1);
23
      continue;
24
    }
25
    current_ = {buf->coalesce(), offset_};
26
    offset_ += sizeof(h) + h.length;
27
    return *this;
28
  }
29
}

1
folly::RecordIOWriter writer(folly::File("events.log", O_WRONLY | O_APPEND | O_CREAT));
2

3
void OnEvent(const Event& e) {
4
  auto buf = SerializeToIOBuf(e);
5
  writer.write(std::move(buf));
6
}

1
folly::RecordIOReader reader(folly::File("events.log", O_RDONLY));
2
size_t valid = 0, recovered = 0;
3
for (auto rec : reader) {
4
  if (rec.first.empty()) continue;
5
  Replay(rec.first);
6
  ++valid;
7
}
8
LOG(INFO) << "valid records: " << valid;

1
// 1. fsync 없이 곧바로 process 종료 가능한 코드
2
writer.write(buf);
3
exit(0);   // page cache가 disk에 안 갔을 수 있음
4
// → fsync(fd) 또는 graceful shutdown
5

6
// 2. record가 너무 작음 (수 byte)
7
for (int i = 0; i < 1e6; ++i) {
8
  writer.write(folly::IOBuf::wrapBuffer(&i, sizeof(i)));
9
}
10
// → 16-byte header overhead가 80%. 묶어서 write 또는 다른 포맷.
11

12
// 3. reader가 매 read마다 새 File 열기
13
for (auto& path : files) {
14
  RecordIOReader r(File(path, O_RDONLY));
15
  for (auto rec : r) { ... }
16
}
17
// → 정상 사용. 단 동일 파일을 반복 열면 page cache는 유지되지만 syscall은 누적.
18

19
// 4. checksum 무시
20
for (auto rec : reader) {
21
  if (rec.first.empty()) continue;   // forward search 결과는 OK
22
  // 하지만 reader가 그저 raw read만 한다면 checksum 검증 skipping 위험
23
}