After upgrading to syncthing 1.5.0, it started upgrading the database from v1.3.4 to v.1.50. This server (odroid hc2) was not doing anything else and including swap had a little less than 4GB available. After running non-stop for 4 days, it crashed with out of memory. Now I know I have a large collection of files, but this seems a bit extreme. Is there something I can do to help this checking of the database?
May 27 10:01:44 <snip>: [start] INFO: syncthing v1.5.0 "Fermium Flea" (go1.13.10 linux-arm) teamcity@build.syncthing.net 2020-04-21 >
May 27 10:01:44 <snip>: [start] INFO: Using large-database tuning
May 27 10:12:28 <snip>: [ZJ3DQ] INFO: My ID: <snip>
May 27 10:12:29 <snip>: [ZJ3DQ] INFO: Single thread SHA256 performance is 51 MB/s using minio/sha256-simd (49 MB/s using crypto/sha2>
May 27 10:12:30 <snip>: [ZJ3DQ] INFO: Hashing performance is 12.17 MB/s
May 27 10:12:30 <snip>: [ZJ3DQ] INFO: Detected upgrade from v1.3.4 to v1.5.0
May 27 10:12:30 <snip>: [ZJ3DQ] INFO: Checking db due to upgrade - this may take a while...
Jun 01 19:13:47 <snip>: runtime: out of memory: cannot allocate 238419968-byte block (2620260352 in use)
Jun 01 19:13:47 <snip>: fatal error: out of memory
Jun 01 19:13:47 <snip>: [monitor] WARNING: Panic detected, writing to "/mnt/syncthing-home/.config/syncthing/panic-20200601-191347.l>
Jun 01 19:13:47 <snip>: [monitor] WARNING: Please check for existing issues with similar panic message at https://github.com/syncthi>
Jun 01 19:13:47 <snip>: [monitor] WARNING: If no issue with similar panic message exists, please create a new issue with the panic l>
Jun 01 19:13:49 <snip>: [monitor] INFO: Reporting crash found in panic-20200601-191347.log (report ID db3a53f6) ...
Here is the panic log if that helps:
[start] INFO: syncthing v1.5.0 "Fermium Flea" (go1.13.10 linux-arm) teamcity@build.syncthing.net 2020-04-21 20:45:03 UTC
[start] INFO: Using large-database tuning
[ZJ3DQ] INFO: My ID: ZJ3DQKF-PK5YLTH-Z2B6GNE-IIB7JK3-2QBGRRT-FUHV7PC-32OXBH2-OZJBZAP
[ZJ3DQ] INFO: Single thread SHA256 performance is 51 MB/s using minio/sha256-simd (49 MB/s using crypto/sha256).
[ZJ3DQ] INFO: Hashing performance is 12.17 MB/s
[ZJ3DQ] INFO: Detected upgrade from v1.3.4 to v1.5.0
[ZJ3DQ] INFO: Checking db due to upgrade - this may take a while...
...
Panic at 2020-06-01T19:13:47+02:00
fatal error: out of memory
runtime stack:
runtime.throw(0x8a1f4d, 0xd)
/usr/local/go/src/runtime/panic.go:774 +0x5c
runtime.largeAlloc(0xe360000, 0x70001, 0xa4aa3550)
/usr/local/go/src/runtime/malloc.go:1140 +0x124
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:1033 +0x38
runtime.systemstack(0x1584000)
/usr/local/go/src/runtime/asm_arm.s:354 +0x84
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1146
goroutine 22 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_arm.s:298 +0x4 fp=0x5c28f83c sp=0x5c28f838 pc=0x71754
runtime.mallocgc(0xe360000, 0x805ef8, 0x1, 0x11)
/usr/local/go/src/runtime/malloc.go:1032 +0x8d0 fp=0x5c28f8a4 sp=0x5c28f83c pc=0x1bbb0
runtime.newarray(0x805ef8, 0x88000, 0x95ba5ca8)
/usr/local/go/src/runtime/malloc.go:1173 +0x68 fp=0x5c28f8b8 sp=0x5c28f8a4 pc=0x1bf4c
runtime.makeBucketArray(0x7c6ee8, 0x13, 0x0, 0x9c9fdd74, 0x21138)
/usr/local/go/src/runtime/map.go:362 +0x178 fp=0x5c28f8d4 sp=0x5c28f8b8 pc=0x1cf94
runtime.hashGrow(0x7c6ee8, 0x24198720)
/usr/local/go/src/runtime/map.go:1033 +0x80 fp=0x5c28f8f8 sp=0x5c28f8d4 pc=0x1e9d8
runtime.mapassign_fast64(0x7c6ee8, 0x24198720, 0x9fde80, 0x0, 0x8ac59008)
/usr/local/go/src/runtime/map_fast64.go:156 +0x10c fp=0x5c28f914 sp=0x5c28f8f8 pc=0x20f5c
github.com/syndtr/goleveldb/leveldb.(*versionStaging).commit(0x5c28f9d8, 0x15f41c0)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/version.go:446 +0x188 fp=0x5c28f9c8 sp=0x5c28f914 pc=0x4f715c
github.com/syndtr/goleveldb/leveldb.(*version).spawn(0x2a2eda40, 0x15f41c0, 0x43a01, 0x627fc)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/version.go:282 +0x54 fp=0x5c28f9e8 sp=0x5c28f9c8 pc=0x4f61d0
github.com/syndtr/goleveldb/leveldb.(*session).commit(0x17da5a0, 0x15f41c0, 0x14a99201, 0x0, 0x0)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/session.go:216 +0x78 fp=0x5c28fa50 sp=0x5c28f9e8 pc=0x4eb8f4
github.com/syndtr/goleveldb/leveldb.(*DB).compactionCommit.func1(0x14eb9f8, 0x14a99201, 0x14eb9f8)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:264 +0x30 fp=0x5c28fa68 sp=0x5c28fa50 pc=0x4fa1cc
github.com/syndtr/goleveldb/leveldb.(*compactionTransactFunc).run(0x14e2688, 0x14eb9f8, 0x14a99230, 0x0)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:242 +0x28 fp=0x5c28fa78 sp=0x5c28fa68 pc=0x4dc96c
github.com/syndtr/goleveldb/leveldb.(*DB).compactionTransact(0x14ac120, 0x14eba00, 0xc, 0xb77518, 0x14e2688)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:186 +0x12c fp=0x5c28fbe8 sp=0x5c28fa78 pc=0x4dc194
github.com/syndtr/goleveldb/leveldb.(*DB).compactionTransactFunc(...)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:253
github.com/syndtr/goleveldb/leveldb.(*DB).compactionCommit(0x14ac120, 0x89a450, 0x5, 0x15f41c0)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:263 +0x124 fp=0x5c28fc38 sp=0x5c28fbe8 pc=0x4dcb0c
github.com/syndtr/goleveldb/leveldb.(*DB).tableCompaction(0x14ac120, 0x22cf2090, 0x1)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:584 +0x534 fp=0x5c28fdf4 sp=0x5c28fc38 pc=0x4deb7c
github.com/syndtr/goleveldb/leveldb.(*DB).tableRangeCompaction(0x14ac120, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:628 +0x26c fp=0x5c28fe60 sp=0x5c28fdf4 pc=0x4df5e8
github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0x14ac120)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:857 +0x3b0 fp=0x5c28ffe4 sp=0x5c28fe60 pc=0x4e0674
runtime.goexit()
/usr/local/go/src/runtime/asm_arm.s:868 +0x4 fp=0x5c28ffe4 sp=0x5c28ffe4 pc=0x7348c
created by github.com/syndtr/goleveldb/leveldb.openDB
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db.go:155 +0x40c
goroutine 1 [semacquire, 7738 minutes]:
sync.runtime_SemacquireMutex(0xd64460c, 0x0, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x34
sync.(*RWMutex).RLock(0xd644600)
/usr/local/go/src/sync/rwmutex.go:50 +0x5c
github.com/syncthing/syncthing/lib/db.(*Lowlevel).getMetaAndCheck(0x15649c0, 0xd644460, 0x15, 0x0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/db/lowlevel.go:676 +0x30
github.com/syncthing/syncthing/lib/db.(*Lowlevel).CheckRepair(0x15649c0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/db/lowlevel.go:671 +0x5c
github.com/syncthing/syncthing/lib/syncthing.(*App).startup(0xd6a80c0, 0x70a90, 0xd6377dc)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/syncthing/syncthing.go:249 +0x2358
github.com/syncthing/syncthing/lib/syncthing.(*App).Start(0xd6a80c0, 0xa6a838, 0xd6a80c0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/syncthing/syncthing.go:113 +0x1c
main.syncthingMain(0x1428240, 0x29, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:706 +0x690
main.main()
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:438 +0x3e0
goroutine 5 [syscall, 7752 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:147 +0x130
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x14
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:29 +0x30
goroutine 6 [chan receive, 7752 minutes]:
github.com/syncthing/notify.(*nonrecursiveTree).dispatch(0x1425280, 0x1425200)
/go/pkg/mod/github.com/syncthing/notify@v0.0.0-20190709140112-69c7a957d3e2/tree_nonrecursive.go:36 +0x94
created by github.com/syncthing/notify.newNonrecursiveTree
/go/pkg/mod/github.com/syncthing/notify@v0.0.0-20190709140112-69c7a957d3e2/tree_nonrecursive.go:29 +0xb8
goroutine 7 [chan receive, 7752 minutes]:
github.com/syncthing/notify.(*nonrecursiveTree).internal(0x1425280, 0x1425240)
/go/pkg/mod/github.com/syncthing/notify@v0.0.0-20190709140112-69c7a957d3e2/tree_nonrecursive.go:81 +0x34
created by github.com/syncthing/notify.newNonrecursiveTree
/go/pkg/mod/github.com/syncthing/notify@v0.0.0-20190709140112-69c7a957d3e2/tree_nonrecursive.go:30 +0xdc
goroutine 50 [select, 7742 minutes]:
github.com/syncthing/syncthing/lib/events.(*logger).serve(0x14ec8c0, 0xb7ade0, 0x14deb80)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/events/events.go:276 +0xec
github.com/syncthing/syncthing/lib/util.AsService.func1(0xb7ade0, 0x14deb80, 0x15e4080, 0x0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/util/utils.go:183 +0x2c
github.com/syncthing/syncthing/lib/util.(*service).Serve(0x14dc7b0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/util/utils.go:247 +0x124
created by main.syncthingMain
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:601 +0x234
goroutine 51 [select]:
github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0x15f3080)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/util/buffer_pool.go:206 +0xe8
created by github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/util/buffer_pool.go:237 +0x128
goroutine 52 [select, 4 minutes]:
github.com/syndtr/goleveldb/leveldb.(*session).refLoop(0x17da5a0)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/session_util.go:189 +0x368
created by github.com/syndtr/goleveldb/leveldb.newSession
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/session.go:93 +0x228
goroutine 20 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0x14ac120)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:91 +0x88
created by github.com/syndtr/goleveldb/leveldb.openDB
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db.go:148 +0x2f0
goroutine 21 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0x14ac120)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_state.go:101 +0xac
created by github.com/syndtr/goleveldb/leveldb.openDB
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db.go:149 +0x30c
goroutine 23 [select, 7712 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0x14ac120)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:773 +0x110
created by github.com/syndtr/goleveldb/leveldb.openDB
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db.go:156 +0x428
goroutine 38 [chan receive, 7742 minutes]:
main.setupSignalHandling.func1(0x1564a00, 0xd6a80c0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:734 +0x24
created by main.setupSignalHandling
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:733 +0x94
goroutine 39 [select, 7712 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).compTriggerRange(0x14ac120, 0xc1f37c0, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_compaction.go:748 +0x254
github.com/syndtr/goleveldb/leveldb.(*DB).CompactRange(0x14ac120, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190923125748-758128399b1d/leveldb/db_write.go:435 +0x2b0
github.com/syncthing/syncthing/lib/db/backend.(*leveldbBackend).Compact(0x29cea008, 0x0, 0x0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/db/backend/leveldb_backend.go:107 +0x98
github.com/syncthing/syncthing/lib/db.(*Lowlevel).gcIndirect(0x15649c0, 0xb7ade0, 0x14e49e0, 0x0, 0x0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/db/lowlevel.go:665 +0x54c
github.com/syncthing/syncthing/lib/db.(*Lowlevel).gcRunner(0x15649c0, 0xb7ade0, 0x14e49e0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/db/lowlevel.go:540 +0x1d0
github.com/syncthing/syncthing/lib/util.AsService.func1(0xb7ade0, 0x14e49e0, 0x22d01900, 0x0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/util/utils.go:183 +0x2c
github.com/syncthing/syncthing/lib/util.(*service).Serve(0xd646570)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/util/utils.go:247 +0x124
github.com/thejerf/suture.(*Supervisor).runService.func1(0x17da140, 0x0, 0xa6bea2e0, 0xd646570)
/go/pkg/mod/github.com/thejerf/suture@v3.0.2+incompatible/supervisor.go:600 +0x34
created by github.com/thejerf/suture.(*Supervisor).runService
/go/pkg/mod/github.com/thejerf/suture@v3.0.2+incompatible/supervisor.go:588 +0x44
goroutine 132 [select, 57 minutes]:
main.autoUpgrade(0xb827c8, 0x17ee000, 0xd6a80c0, 0xb7b020, 0x14ec8c0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:856 +0xf8
created by main.syncthingMain
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:681 +0xa74
goroutine 25 [chan receive, 7742 minutes]:
main.setupSignalHandling.func2(0x14eca00, 0xd6a80c0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:743 +0x24
created by main.setupSignalHandling
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:742 +0x128
goroutine 26 [sleep]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:310
time.Sleep(0x540be400, 0x2)
/usr/local/go/src/runtime/time.go:105 +0x158
main.standbyMonitor(0xd6a80c0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:822 +0x48
created by main.syncthingMain
/opt/tcagent/syncthing-4-work/174e136266f8a219/cmd/syncthing/main.go:703 +0x790
goroutine 27 [select, 7742 minutes]:
github.com/thejerf/suture.(*Supervisor).Serve(0x14ba780)
/go/pkg/mod/github.com/thejerf/suture@v3.0.2+incompatible/supervisor.go:454 +0x270
created by github.com/thejerf/suture.(*Supervisor).ServeBackground
/go/pkg/mod/github.com/thejerf/suture@v3.0.2+incompatible/supervisor.go:407 +0x2c
goroutine 28 [select, 7742 minutes]:
github.com/thejerf/suture.(*Supervisor).Serve(0x17da140)
/go/pkg/mod/github.com/thejerf/suture@v3.0.2+incompatible/supervisor.go:454 +0x270
github.com/thejerf/suture.(*Supervisor).runService.func1(0x14ba780, 0x0, 0xb77398, 0x15649c0)
/go/pkg/mod/github.com/thejerf/suture@v3.0.2+incompatible/supervisor.go:600 +0x34
created by github.com/thejerf/suture.(*Supervisor).runService
/go/pkg/mod/github.com/thejerf/suture@v3.0.2+incompatible/supervisor.go:588 +0x44
goroutine 29 [chan receive, 7742 minutes]:
github.com/syncthing/syncthing/lib/events.(*bufferedSubscription).pollingLoop(0xd63c7e0)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/events/events.go:467 +0x6c
created by github.com/syncthing/syncthing/lib/events.NewBufferedSubscription
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/events/events.go:462 +0x130
goroutine 30 [chan receive, 7742 minutes]:
github.com/syncthing/syncthing/lib/events.(*bufferedSubscription).pollingLoop(0xd63c840)
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/events/events.go:467 +0x6c
created by github.com/syncthing/syncthing/lib/events.NewBufferedSubscription
/opt/tcagent/syncthing-4-work/174e136266f8a219/lib/events/events.go:462 +0x130
3 posts - 2 participants