Configuring NGINX for large repos - could really use some help please

I’ve deployed Seafile Server Professional (latest release) on Ubuntu Server 18.04 behind nginx, created and populated several repos / Libraries and am attempting to download and sync them externally using the CLI client on another Ubuntu 18.04 box.

My repos vary in size between 7.7 GB and 140GB.
Repos below 50gb are downloading without issue, but anything above that size is returning an ‘unknown error’, with log entries as the below example.

[04/24/20 10:43:19] http-tx-mgr.c(4145): Bad response code for POST https://seafile.mydomain.com/seafhttp/repo/60b9212b-9be3-4525-8d3d-3d17f917adb8/pack-fs/: 0.
[04/24/20 10:43:19] http-tx-mgr.c(4572): Failed to get fs objects for repo 60b9212b on server https://seafile.mydomain.com.
[04/24/20 10:43:19] http-tx-mgr.c(1157): Transfer repo '60b9212b': ('normal', 'fs') --> ('error', 'finished')
[04/24/20 10:43:19] clone-mgr.c(697): Transition clone state for 60b9212b from [fetch] to [error]: Unknown error.

Based on my reading so far, I believe there is an issue with my nginx configuration, which is below:

log_format seafileformat '$http_x_forwarded_for $remote_addr [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $upstream_response_time';

server {

    server_name seafile.mydomain.com;

    proxy_set_header X-Forwarded-For $remote_addr;

    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";

    server_tokens off;

    location / {

        proxy_pass         http://127.0.0.1:8000;

        proxy_set_header   Host $host;

        proxy_set_header   X-Real-IP $remote_addr;

        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;

        proxy_set_header   X-Forwarded-Host $server_name;

        proxy_set_header   X-Forwarded-Proto https;

        proxy_read_timeout 1200s;

        proxy_buffer_size 16k;

        proxy_busy_buffers_size 16k;

        # used for view/edit office file via Office Online Server

        client_max_body_size 0;

        # logs

        access_log      /var/log/nginx/seahub.access.log seafileformat;

        error_log       /var/log/nginx/seahub.error.log;

    }

    location /seafhttp {

         rewrite ^/seafhttp(.*)$ $1 break;

         proxy_pass http://127.0.0.1:8082;

         client_max_body_size 0;

         proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;

         proxy_connect_timeout  36000s;

         proxy_read_timeout  36000s;

         proxy_request_buffering off;

         send_timeout 36000s;

         proxy_request_buffering off;

         proxy_buffer_size 16k;

         proxy_busy_buffers_size 16k;

         access_log      /var/log/nginx/seafhttp.access.log seafileformat;

         error_log       /var/log/nginx/seafhttp.error.log;

    }

    location /media {

        root /opt/seafile/seafile-server-latest/seahub;

    }

    location /seafdav {

        fastcgi_pass    127.0.0.1:8080;

        fastcgi_param   SCRIPT_FILENAME     $document_root$fastcgi_script_name;

        fastcgi_param   PATH_INFO           $fastcgi_script_name;

        fastcgi_param   SERVER_PROTOCOL     $server_protocol;

        fastcgi_param   QUERY_STRING        $query_string;

        fastcgi_param   REQUEST_METHOD      $request_method;

        fastcgi_param   CONTENT_TYPE        $content_type;

        fastcgi_param   CONTENT_LENGTH      $content_length;

        fastcgi_param   SERVER_ADDR         $server_addr;

        fastcgi_param   SERVER_PORT         $server_port;

        fastcgi_param   SERVER_NAME         $server_name;

        fastcgi_param   REMOTE_ADDR         $remote_addr;

        client_max_body_size 0;

        access_log      /var/log/nginx/seafdav.access.log seafileformat;

        error_log       /var/log/nginx/seafdav.error.log;

    }

    listen 443 ssl; # managed by Certbot

    ssl_certificate /etc/letsencrypt/live/seafile.mydomain.com/fullchain.pem; # managed by Certbot

    ssl_certificate_key /etc/letsencrypt/live/seafile.mydomain.com/privkey.pem; # managed by Certbot

    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot

    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot

}

server {

    if ($host = seafile.mydomain.com) {

        return 301 https://$host$request_uri;

    } # managed by Certbot

    listen 80;

    server_name seafile.mydomain.com;

    return 404; # managed by Certbot

}

I’m a little out of my league here so would appreciate any pointers from more experienced people.

1 Like

Still digging on this, and still desperately need any help from the community.

It looks to my noobish eyes to be a timeout on the problem repos, based on seafhttp.access.log and comparing:

Repo that syncs

- clientipaddress [28/Apr/2020:14:19:54 +0100] "GET /seafhttp/protocol-version HTTP/1.1" 200 14 "-" "Seafile/7.0.4 (Linux)" 0.000
- clientipaddress [28/Apr/2020:14:19:54 +0100] "GET /seafhttp/repo/a04f521b-7014-43dd-a1ed-a107ad2fb087/commit/HEAD HTTP/1.1" 200 81 "-" "Seafile/7.0.4 (Linux)" 0.000
- clientipaddress [28/Apr/2020:14:19:54 +0100] "GET /seafhttp/repo/a04f521b-7014-43dd-a1ed-a107ad2fb087/permission-check/?op=download&client_id=c355f9af8b489f61719371c0b26369cc1a25147d&client_name=unknown HTTP/1.1" 200 0 "-" "Seafile/7.0.4 (Linux)" 0.012
- clientipaddress [28/Apr/2020:14:19:54 +0100] "GET /seafhttp/repo/a04f521b-7014-43dd-a1ed-a107ad2fb087/commit/b4351efa0ff5ca289061fed1f0196d95f3a23a26 HTTP/1.1" 200 554 "-" "Seafile/7.0.4 (Linux)" 0.000
- clientipaddress [28/Apr/2020:14:19:54 +0100] "GET /seafhttp/repo/a04f521b-7014-43dd-a1ed-a107ad2fb087/fs-id-list/?server-head=b4351efa0ff5ca289061fed1f0196d95f3a23a26 HTTP/1.1" 200 130 "-" "Seafile/7.0.4 (Linux)" 0.004
- clientipaddress [28/Apr/2020:14:19:54 +0100] "POST /seafhttp/repo/a04f521b-7014-43dd-a1ed-a107ad2fb087/pack-fs/ HTTP/1.1" 200 59942 "-" "Seafile/7.0.4 (Linux)" 0.000
- clientipaddress [28/Apr/2020:14:19:58 +0100] "GET /seafhttp/repo/a04f521b-7014-43dd-a1ed-a107ad2fb087/block/db8c77ed4c8e5d497d338a6a97c2d37e0fbd7da5 HTTP/1.1" 200 9360955 "-" "Seafile/7.0.4 (Linux)" 0.028
- clientipaddress [28/Apr/2020:14:20:01 +0100] "GET /seafhttp/repo/a04f521b-7014-43dd-a1ed-a107ad2fb087/block/9ac25f3bc8040f9c001a550a89047252f8a91204 HTTP/1.1" 200 6866633 "-" "Seafile/7.0.4 (Linux)" 0.016
- clientipaddress [28/Apr/2020:14:20:05 +0100] "GET /seafhttp/repo/a04f521b-7014-43dd-a1ed-a107ad2fb087/block/a26d87ddd4b53c9e3b223199b6e78f638d12af98 HTTP/1.1" 200 10485760 "-" "Seafile/7.0.4 (Linux)" 0.080
- etc.

Repo that fails

- clientipaddress [28/Apr/2020:14:20:59 +0100] "GET /seafhttp/protocol-version HTTP/1.1" 200 14 "-" "Seafile/7.0.4 (Linux)" 0.000
- clientipaddress [28/Apr/2020:14:20:59 +0100] "GET /seafhttp/repo/d596f397-2163-4dbb-b1e1-ce1130b68ea2/commit/HEAD HTTP/1.1" 200 81 "-" "Seafile/7.0.4 (Linux)" 0.004
- clientipaddress [28/Apr/2020:14:20:59 +0100] "GET /seafhttp/repo/d596f397-2163-4dbb-b1e1-ce1130b68ea2/permission-check/?op=download&client_id=c355f9af8b489f61719371c0b26369cc1a25147d&client_name=unknown HTTP/1.1" 200 0 "-" "Seafile/7.0.4 (Linux)" 0.016
- clientipaddress [28/Apr/2020:14:20:59 +0100] "GET /seafhttp/repo/d596f397-2163-4dbb-b1e1-ce1130b68ea2/commit/49d6c281eb255f123c4cc43e6f747dd29e18acd1 HTTP/1.1" 200 526 "-" "Seafile/7.0.4 (Linux)" 0.000
- clientipaddress [28/Apr/2020:14:20:59 +0100] "GET /seafhttp/repo/d596f397-2163-4dbb-b1e1-ce1130b68ea2/fs-id-list/?server-head=49d6c281eb255f123c4cc43e6f747dd29e18acd1 HTTP/1.1" 200 2624 "-" "Seafile/7.0.4 (Linux)" 0.004
- clientipaddress [28/Apr/2020:14:21:59 +0100] "POST /seafhttp/repo/d596f397-2163-4dbb-b1e1-ce1130b68ea2/pack-fs/ HTTP/1.1" 408 0 "-" "Seafile/7.0.4 (Linux)"

seaf-cli log

[04/28/20 14:20:59] clone-mgr.c(680): Transition clone state for d596f397 from [init] to [check server].
[04/28/20 14:20:59] Couldn't prepare query, error:1->'no such table: Certs'
        SELECT cert FROM Certs;
[04/28/20 14:20:59] clone-mgr.c(680): Transition clone state for d596f397 from [check server] to [fetch].
[04/28/20 14:20:59] http-tx-mgr.c(1157): Transfer repo 'd596f397': ('normal', 'init') --> ('normal', 'check')
[04/28/20 14:20:59] http-tx-mgr.c(1157): Transfer repo 'd596f397': ('normal', 'check') --> ('normal', 'commit')
[04/28/20 14:20:59] http-tx-mgr.c(1157): Transfer repo 'd596f397': ('normal', 'commit') --> ('normal', 'fs')
[04/28/20 14:21:59] http-tx-mgr.c(4145): Bad response code for POST https://seafile.mydomain.com/seafhttp/repo/d596f397-2163-4dbb-b1e1-ce1130b68ea2/pack-fs/: 0.
[04/28/20 14:21:59] http-tx-mgr.c(4572): Failed to get fs objects for repo d596f397 on server https://seafile.mydomain.com.
[04/28/20 14:21:59] http-tx-mgr.c(1157): Transfer repo 'd596f397': ('normal', 'fs') --> ('error', 'finished')
[04/28/20 14:21:59] clone-mgr.c(697): Transition clone state for d596f397 from [fetch] to [error]: Unknown error.

I’ve been working on this too long to give up, but have yet to find any help online.

Also of note:

  1. I’ve run the repository checker tool and garbage collection to check for issues and clean-up the server.
  2. At least one of the errored repos can be downloaded using the Windows Sync client with the same account credentials.

I’m not a Seafile expert by any means, but it would be good to rule out any resource issues.

Some questions/things to consider:

  • Do the repos consist of lots of small files or a few really big ones?
  • Have your looked at other logs in the system? dmesg?
  • Could it be that you are hitting the resource limits imposed by limits.conf?
  • It might also be worth lookingat the memory usage when downloading one of these repos. Could it be that you run out of memory and the kernel OOM killer kicks in?

Also, have you tried increasing the various nginx timeouts?

Thanks for responding - I’m sure you’re in a better position than me as I’m new to seafile and nginx as well.

  • Repos are varying numbers of small and some large files. There seems to be no pattern between those that are having issues and those that are not (one of the stuck repos has little more than 60 small files at root or in 1 level of sub folders

  • I had not thought to look in dmesg (my linux knowledge is rather patchy as it’s built on resolving issues on previous specific installations rather than general long term experience). I will certainly do so in the morning.

  • I haven’t run across a limits.conf file yet on my installation, but having quickly looked at the link you supplied I’m not sure how I would determine if this was the issue.
    I’m uploading to the Seafile server from the local lan using a Seadrive windows client and robocopy which has not seemingly had any issues; once I moved from a very overtaxed windows box to one with more resources at least. Files upload correctly and are accessible through the web interface. It’s the remote CLI client on an Ubuntu box trying to download the files that is having problems, typically failing to progress beyond a partial download of the file list now **

  • I’ll take a look at memory usage in the morning also.

  • I have fiddled with the nginx overall timeouts today, as well as:

    • removing the history for the repos (not required for these repos)
    • clearing down and rebuilding the repos after finding an issue with the upload robocopy which could have caused the seafile client to believe that the root library had changed (folder attribute related).
    • resetting the seafile client configuration and data and restarting the downloads

Currently I’m in the process of re-uploading the files to all repos and have all repos (some empty currently) showing as ‘synchronised’ on the seafile client. The repos will continue to fill overnight and I’m hoping to see downloading / synchronised for them at the remote client end tomorrow.
I have seen some data transfer timeouts logged at the client since I restarted this process but this may be due to the current load on the seafile server.

I seem to have staggered past the ‘unknown error’ at the seaf-cli end yesterday by extending the nginx timeouts, however some repos are still failing to download this morning, stuck at “downloading file list”.
I’m now getting a proper error in the client-side logs.

Client logs:

[04/30/20 08:47:00] http-tx-mgr.c(1157): Transfer repo '54d15b3e': ('normal', 'fs') --> ('error', 'finished')
[04/30/20 08:47:00] sync-mgr.c(621): Repo 'RemoteBackupScripts' sync state transition from downloading to 'error': 'Data transfer timed out. Please check network or firewall'.
[04/30/20 08:47:00] Couldn't prepare query, error:1->'no such table: Certs'
        SELECT cert FROM Certs;
[04/30/20 08:47:00] sync-mgr.c(582): Repo 'RemoteBackupScripts' sync state transition from 'initializing' to 'downloading'.
[04/30/20 08:47:00] http-tx-mgr.c(1157): Transfer repo '54d15b3e': ('normal', 'init') --> ('normal', 'check')
[04/30/20 08:47:00] http-tx-mgr.c(1157): Transfer repo '54d15b3e': ('normal', 'check') --> ('normal', 'commit')
[04/30/20 08:47:00] http-tx-mgr.c(1157): Transfer repo '54d15b3e': ('normal', 'commit') --> ('normal', 'fs')
[04/30/20 08:47:10] Couldn't prepare query, error:1->'no such table: Certs'
        SELECT cert FROM Certs;
[04/30/20 08:50:45] http-tx-mgr.c(1045): libcurl failed to POST https://seafile.mydomain.com/seafhttp/repo/7f6a1a7e-25ec-4cf4-a896-5d26c5a11a02/pack-fs/: Timeout was reached.
[04/30/20 08:50:45] http-tx-mgr.c(4572): Failed to get fs objects for repo 7f6a1a7e on server https://seafile.mydomain.com.
[04/30/20 08:50:45] http-tx-mgr.c(1157): Transfer repo '7f6a1a7e': ('normal', 'fs') --> ('error', 'finished')
[04/30/20 08:50:45] sync-mgr.c(621): Repo 'Company' sync state transition from downloading to 'error': 'Data transfer timed out. Please check network or firewall'.
[04/30/20 08:50:45] sync-mgr.c(582): Repo 'Company' sync state transition from 'initializing' to 'downloading'.
[04/30/20 08:50:45] http-tx-mgr.c(1157): Transfer repo '7f6a1a7e': ('normal', 'init') --> ('normal', 'check')
[04/30/20 08:50:45] http-tx-mgr.c(1157): Transfer repo '7f6a1a7e': ('normal', 'check') --> ('normal', 'commit')
[04/30/20 08:50:45] http-tx-mgr.c(1157): Transfer repo '7f6a1a7e': ('normal', 'commit') --> ('normal', 'fs')
[04/30/20 08:51:10] Couldn't prepare query, error:1->'no such table: Certs'
        SELECT cert FROM Certs;
[04/30/20 08:51:33] http-tx-mgr.c(1045): libcurl failed to POST https://seafile.mydomain.com/seafhttp/repo/f0707b23-81b7-49b5-9da9-588efa62705a/pack-fs/: Timeout was reached.
[04/30/20 08:51:33] http-tx-mgr.c(4572): Failed to get fs objects for repo f0707b23 on server https://seafile.mydomain.com.
[04/30/20 08:51:34] http-tx-mgr.c(1157): Transfer repo 'f0707b23': ('normal', 'fs') --> ('error', 'finished')
[04/30/20 08:51:34] sync-mgr.c(621): Repo 'SubversionRepositories' sync state transition from downloading to 'error': 'Data transfer timed out. Please check network or firewall'.
[04/30/20 08:51:34] sync-mgr.c(582): Repo 'SubversionRepositories' sync state transition from 'initializing' to 'downloading'.
[04/30/20 08:51:34] http-tx-mgr.c(1157): Transfer repo 'f0707b23': ('normal', 'init') --> ('normal', 'check')
[04/30/20 08:51:34] http-tx-mgr.c(1157): Transfer repo 'f0707b23': ('normal', 'check') --> ('normal', 'commit')

The ‘Company’ repo is massive (currently 37406 / 124.0GB) however ‘RemoteBackupScripts’ is only small (43 / 124.8KB).

I’ve had a look in dmesg this morning but if I’m honest I don’t know what I should be looking for, nor how to establish whether it’s a resourcing issue at the server-side.
It seems odd to me that some repos completed successfully (including one that is 21.5GB) while others have not, although I am still trying to load the bigger repos.
I’m tempted to spin up a windows gui client to see if that has the same issues.

EDIT
Using the windows sync client and the same account credentials as the failing seaf-cli I’ve been able to sync two of the troublesome repos without issue. Windows client on a different (200/20) connection, to the Linux client (30/10), but both passing through nginx.