From ffeca193c0f2936e064b503e85449669ee6b89e2 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Thu, 27 Jun 2019 16:12:59 +0200 Subject: [PATCH 1/6] build_library: retry losetup up to 10 seconds Sometimes we see such failures during Jenkins runs. ``` losetup: /mnt/host/source/tmp/...../disk_image.bin: failed to set up loop device: Resource temporarily unavailable ``` It's really hard to know which race condition it is. To workaround it, in case that losetup fails, make it retry up to 10 seconds, once in every second. That could reduce the number of the loopback device errors in the build chain. Inspired by https://chromium-review.googlesource.com/c/chromiumos/platform/crosutils/+/264063/ --- build_library/disk_util | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index 0e79976860..9a54ed0774 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -11,6 +11,7 @@ import re import subprocess import sys import tempfile +import time import uuid # First sector we can use. @@ -433,17 +434,23 @@ def FormatFat(part, device): def PartitionLoop(options, partition): """Allocate (and automatically free) loop devices for a partition.""" - loop_dev = subprocess.check_output(['sudo', 'losetup', - '--offset', str(partition['first_byte']), - '--sizelimit', str(partition['bytes']), - '--find', '--show', options.disk_image]) - loop_dev = loop_dev.strip() + attempts = 0 - try: - yield loop_dev - finally: - Sudo(['losetup', '--detach', loop_dev]) + while attempts < 10: + try: + loop_dev = subprocess.check_output(['sudo', 'losetup', + '--offset', str(partition['first_byte']), + '--sizelimit', str(partition['bytes']), + '--find', '--show', options.disk_image]) + loop_dev = loop_dev.strip() + yield loop_dev + + except: + attempts += 1 + time.sleep(1) + finally: + Sudo(['losetup', '--detach', loop_dev]) def FormatPartition(options, part): print "Formatting partition %s (%s) as %s" % ( From a8cafdb31cda569271f76a0759cc4aafb53ea13e Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Fri, 28 Jun 2019 17:06:19 +0200 Subject: [PATCH 2/6] build_library: make a loop only for losetup in PartitionLoop Fix a bug in PartitionLoop that ends up returning a false string, which results in FormatPartition failing. --- build_library/disk_util | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index 9a54ed0774..7fd180b23f 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -435,6 +435,7 @@ def PartitionLoop(options, partition): """Allocate (and automatically free) loop devices for a partition.""" attempts = 0 + loop_dev = '' while attempts < 10: try: @@ -443,14 +444,14 @@ def PartitionLoop(options, partition): '--sizelimit', str(partition['bytes']), '--find', '--show', options.disk_image]) loop_dev = loop_dev.strip() - - yield loop_dev - except: attempts += 1 time.sleep(1) - finally: - Sudo(['losetup', '--detach', loop_dev]) + + try: + yield loop_dev + finally: + Sudo(['losetup', '--detach', loop_dev]) def FormatPartition(options, part): print "Formatting partition %s (%s) as %s" % ( From 518f12983f416c96c399faa3ec5a8f96b9173154 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Sat, 29 Jun 2019 09:19:51 +0200 Subject: [PATCH 3/6] build_library: detach after every losetup succeeded To avoid issues with losetup not being cleaned up, we need to call losetup --detach inside the retry loop. --- build_library/disk_util | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index 7fd180b23f..699a173317 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -447,11 +447,10 @@ def PartitionLoop(options, partition): except: attempts += 1 time.sleep(1) + finally: + Sudo(['losetup', '--detach', loop_dev]) - try: - yield loop_dev - finally: - Sudo(['losetup', '--detach', loop_dev]) + yield loop_dev def FormatPartition(options, part): print "Formatting partition %s (%s) as %s" % ( From 5bc76098a53192a50d5342c4fe3d587ae73b202f Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Sat, 29 Jun 2019 10:01:21 +0200 Subject: [PATCH 4/6] build_library: break the attempt loop when losetup succeeds We need to break the attempt loop when losetup succeeds. Also we should not use finally, which is called also for success. --- build_library/disk_util | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index 699a173317..aa7743a5c9 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -444,11 +444,11 @@ def PartitionLoop(options, partition): '--sizelimit', str(partition['bytes']), '--find', '--show', options.disk_image]) loop_dev = loop_dev.strip() + break except: + Sudo(['losetup', '--detach', loop_dev]) attempts += 1 time.sleep(1) - finally: - Sudo(['losetup', '--detach', loop_dev]) yield loop_dev From 06027d670f47a68360e11fa79ba3e2d729d8ab63 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Sat, 29 Jun 2019 16:23:09 +0200 Subject: [PATCH 5/6] build_library: detach only if loop_dev exists --- build_library/disk_util | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_library/disk_util b/build_library/disk_util index aa7743a5c9..eeb6d53ab7 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -446,7 +446,8 @@ def PartitionLoop(options, partition): loop_dev = loop_dev.strip() break except: - Sudo(['losetup', '--detach', loop_dev]) + if loop_dev: + Sudo(['losetup', '--detach', loop_dev]) attempts += 1 time.sleep(1) From ab637f03cf21bbee33d4f837ccd1a1d8c0c03c30 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Sat, 29 Jun 2019 23:03:29 +0200 Subject: [PATCH 6/6] Revert "build_library: retry losetup up to 10 seconds" This reverts commit ffeca193c0f2936e064b503e85449669ee6b89e2, a8cafdb3, 518f1298, 5bc76098, 06027d67 --- build_library/disk_util | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index eeb6d53ab7..0e79976860 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -11,7 +11,6 @@ import re import subprocess import sys import tempfile -import time import uuid # First sector we can use. @@ -434,24 +433,17 @@ def FormatFat(part, device): def PartitionLoop(options, partition): """Allocate (and automatically free) loop devices for a partition.""" - attempts = 0 - loop_dev = '' + loop_dev = subprocess.check_output(['sudo', 'losetup', + '--offset', str(partition['first_byte']), + '--sizelimit', str(partition['bytes']), + '--find', '--show', options.disk_image]) + loop_dev = loop_dev.strip() - while attempts < 10: - try: - loop_dev = subprocess.check_output(['sudo', 'losetup', - '--offset', str(partition['first_byte']), - '--sizelimit', str(partition['bytes']), - '--find', '--show', options.disk_image]) - loop_dev = loop_dev.strip() - break - except: - if loop_dev: - Sudo(['losetup', '--detach', loop_dev]) - attempts += 1 - time.sleep(1) + try: + yield loop_dev + finally: + Sudo(['losetup', '--detach', loop_dev]) - yield loop_dev def FormatPartition(options, part): print "Formatting partition %s (%s) as %s" % (