Skip to content

Commit c9868a0

Browse files
cmagliefacchinm
cmaglie
and
facchinm
authored
Fixed sketchbook+bootloader hex merger when bootloader is in bin format (#744)
* Fixed skethbook+bootloader hex merger. Previous merger doesn't take into account .bin bootloaders and it may have problems if .hex data is not sorted by address. * output a merged sketch+bootloader bin file too * Replace logger with utils.LogIfVerbose Co-authored-by: Martino Facchin <m.facchin@arduino.cc>
1 parent 98dab40 commit c9868a0

File tree

4 files changed

+174
-63
lines changed

4 files changed

+174
-63
lines changed

‎go.mod‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
bou.ke/monkey v1.0.1
77
github.com/GeertJohan/go.rice v1.0.0
88
github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
9-
github.com/arduino/go-paths-helper v1.0.1
9+
github.com/arduino/go-paths-helper v1.2.0
1010
github.com/arduino/go-properties-orderedmap v1.0.0
1111
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
1212
github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b
@@ -25,6 +25,7 @@ require (
2525
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
2626
github.com/juju/testing v0.0.0-20190429233213-dfc56b8c09fc // indirect
2727
github.com/leonelquinteros/gotext v1.4.0
28+
github.com/marcinbor85/gohex v0.0.0-20200531163658-baab2527a9a2
2829
github.com/mattn/go-colorable v0.1.2
2930
github.com/mattn/go-runewidth v0.0.2 // indirect
3031
github.com/miekg/dns v1.0.5 // indirect

‎go.sum‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c h1:agh2JT9
1616
github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c/go.mod h1:HK7SpkEax/3P+0w78iRQx1sz1vCDYYw9RXwHjQTB5i8=
1717
github.com/arduino/go-paths-helper v1.0.1 h1:utYXLM2RfFlc9qp/MJTIYp3t6ux/xM6mWjeEb/WLK4Q=
1818
github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
19+
github.com/arduino/go-paths-helper v1.2.0 h1:qDW93PR5IZUN/jzO4rCtexiwF8P4OIcOmcSgAYLZfY4=
20+
github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
1921
github.com/arduino/go-properties-orderedmap v1.0.0 h1:caaM25TQZKkytoKQUsgqtOVbrM5i8Gb427JmW0KL05s=
2022
github.com/arduino/go-properties-orderedmap v1.0.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
2123
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b h1:9hDi4F2st6dbLC3y4i02zFT5quS4X6iioWifGlVwfy4=
@@ -125,6 +127,8 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
125127
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
126128
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
127129
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
130+
github.com/marcinbor85/gohex v0.0.0-20200531163658-baab2527a9a2 h1:n7R8fUwWZUB2XtyzBNsYNNm9/XgOBj6pvLi7GLMCHtM=
131+
github.com/marcinbor85/gohex v0.0.0-20200531163658-baab2527a9a2/go.mod h1:Pb6XcsXyropB9LNHhnqaknG/vEwYztLkQzVCHv8sQ3M=
128132
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
129133
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
130134
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=

‎legacy/builder/merge_sketch_with_bootloader.go‎

Lines changed: 70 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,15 @@
1616
package builder
1717

1818
import (
19-
"os"
19+
"math"
20+
"strconv"
2021
"strings"
2122

2223
"github.com/arduino/arduino-cli/legacy/builder/constants"
2324
"github.com/arduino/arduino-cli/legacy/builder/types"
25+
"github.com/arduino/arduino-cli/legacy/builder/utils"
2426
"github.com/arduino/go-paths-helper"
27+
"github.com/marcinbor85/gohex"
2528
"github.com/pkg/errors"
2629
)
2730

@@ -36,7 +39,6 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error {
3639
buildPath := ctx.BuildPath
3740
sketch := ctx.Sketch
3841
sketchFileName := sketch.MainFile.Name.Base()
39-
logger := ctx.GetLogger()
4042

4143
sketchInBuildPath := buildPath.Join(sketchFileName + ".hex")
4244
sketchInSubfolder := buildPath.Join(constants.FOLDER_SKETCH, sketchFileName+".hex")
@@ -60,74 +62,92 @@ func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error {
6062

6163
bootloaderPath := buildProperties.GetPath(constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH).Join(constants.FOLDER_BOOTLOADERS, bootloader)
6264
if bootloaderPath.NotExist() {
63-
logger.Fprintln(os.Stdout, constants.LOG_LEVEL_WARN, constants.MSG_BOOTLOADER_FILE_MISSING, bootloaderPath)
65+
utils.LogIfVerbose(constants.LOG_LEVEL_WARN, constants.MSG_BOOTLOADER_FILE_MISSING, bootloaderPath)
6466
return nil
6567
}
6668

6769
mergedSketchPath := builtSketchPath.Parent().Join(sketchFileName + ".with_bootloader.hex")
6870

69-
return merge(builtSketchPath, bootloaderPath, mergedSketchPath)
70-
}
71-
72-
func hexLineOnlyContainsFF(line string) bool {
73-
//:206FE000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFB1
74-
if len(line) <= 11 {
75-
return false
71+
// Ignore merger errors for the first iteration
72+
maximumBinSize := 16000000
73+
if uploadMaxSize, ok := ctx.BuildProperties.GetOk(constants.PROPERTY_UPLOAD_MAX_SIZE); ok {
74+
maximumBinSize, _ = strconv.Atoi(uploadMaxSize)
75+
maximumBinSize *= 2
7676
}
77-
byteArray := []byte(line)
78-
for _, char := range byteArray[9:(len(byteArray) - 2)] {
79-
if char != 'F' {
80-
return false
81-
}
77+
err := merge(builtSketchPath, bootloaderPath, mergedSketchPath, maximumBinSize)
78+
if err != nil {
79+
utils.LogIfVerbose(constants.LOG_LEVEL_INFO, err.Error())
8280
}
83-
return true
84-
}
85-
86-
func extractActualBootloader(bootloader []string) []string {
8781

88-
var realBootloader []string
82+
return nil
83+
}
8984

90-
// skip until we find a line full of FFFFFF (except address and checksum)
91-
for i, row := range bootloader {
92-
if hexLineOnlyContainsFF(row) {
93-
realBootloader = bootloader[i:len(bootloader)]
94-
break
95-
}
85+
func merge(builtSketchPath, bootloaderPath, mergedSketchPath *paths.Path, maximumBinSize int) error {
86+
if bootloaderPath.Ext() == ".bin" {
87+
bootloaderPath = paths.New(strings.TrimSuffix(bootloaderPath.String(), ".bin") + ".hex")
9688
}
9789

98-
// drop all "empty" lines
99-
for i, row := range realBootloader {
100-
if !hexLineOnlyContainsFF(row) {
101-
realBootloader = realBootloader[i:len(realBootloader)]
102-
break
90+
memBoot := gohex.NewMemory()
91+
if bootFile, err := bootloaderPath.Open(); err == nil {
92+
defer bootFile.Close()
93+
if err := memBoot.ParseIntelHex(bootFile); err != nil {
94+
return errors.New(bootFile.Name() + " " + err.Error())
10395
}
96+
} else {
97+
return err
10498
}
10599

106-
if len(realBootloader) == 0 {
107-
// we didn't find any line full of FFFF, thus it's a standalone bootloader
108-
realBootloader = bootloader
100+
memSketch := gohex.NewMemory()
101+
if buildFile, err := builtSketchPath.Open(); err == nil {
102+
defer buildFile.Close()
103+
if err := memSketch.ParseIntelHex(buildFile); err != nil {
104+
return errors.New(buildFile.Name() + " " + err.Error())
105+
}
106+
} else {
107+
return err
109108
}
110109

111-
return realBootloader
112-
}
110+
memMerged := gohex.NewMemory()
111+
initialAddress := uint32(math.MaxUint32)
112+
lastAddress := uint32(0)
113113

114-
func merge(builtSketchPath, bootloaderPath, mergedSketchPath *paths.Path) error {
115-
sketch, err := builtSketchPath.ReadFileAsLines()
116-
if err != nil {
117-
return errors.WithStack(err)
114+
for _, segment := range memBoot.GetDataSegments() {
115+
if err := memMerged.AddBinary(segment.Address, segment.Data); err != nil {
116+
continue
117+
}
118+
if segment.Address < initialAddress {
119+
initialAddress = segment.Address
120+
}
121+
if segment.Address+uint32(len(segment.Data)) > lastAddress {
122+
lastAddress = segment.Address + uint32(len(segment.Data))
123+
}
118124
}
119-
sketch = sketch[:len(sketch)-2]
120-
121-
bootloader, err := bootloaderPath.ReadFileAsLines()
122-
if err != nil {
123-
return errors.WithStack(err)
125+
for _, segment := range memSketch.GetDataSegments() {
126+
if err := memMerged.AddBinary(segment.Address, segment.Data); err != nil {
127+
continue
128+
}
129+
if segment.Address < initialAddress {
130+
initialAddress = segment.Address
131+
}
132+
if segment.Address+uint32(len(segment.Data)) > lastAddress {
133+
lastAddress = segment.Address + uint32(len(segment.Data))
134+
}
124135
}
125136

126-
realBootloader := extractActualBootloader(bootloader)
127-
128-
for _, row := range realBootloader {
129-
sketch = append(sketch, row)
137+
if mergeFile, err := mergedSketchPath.Create(); err == nil {
138+
defer mergeFile.Close()
139+
memMerged.DumpIntelHex(mergeFile, 16)
140+
} else {
141+
return err
130142
}
131143

132-
return mergedSketchPath.WriteFile([]byte(strings.Join(sketch, "\n")))
144+
// Write out a .bin if the addresses doesn't go too far away from origin
145+
// (and consequently produce a very large bin)
146+
size := lastAddress - initialAddress
147+
if size > uint32(maximumBinSize) {
148+
return nil
149+
}
150+
mergedSketchPathBin := paths.New(strings.TrimSuffix(mergedSketchPath.String(), ".hex") + ".bin")
151+
data := memMerged.ToBinary(initialAddress, size, 0xFF)
152+
return mergedSketchPathBin.WriteFile(data)
133153
}

‎legacy/builder/test/merge_sketch_with_bootloader_test.go‎

Lines changed: 98 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package test
1717

1818
import (
19+
"fmt"
1920
"path/filepath"
2021
"strings"
2122
"testing"
@@ -46,8 +47,36 @@ func TestMergeSketchWithBootloader(t *testing.T) {
4647
err := buildPath.Join("sketch").MkdirAll()
4748
NoError(t, err)
4849

49-
fakeSketchHex := "row 1\n" +
50-
"row 2\n"
50+
fakeSketchHex := `:100000000C9434000C9446000C9446000C9446006A
51+
:100010000C9446000C9446000C9446000C94460048
52+
:100020000C9446000C9446000C9446000C94460038
53+
:100030000C9446000C9446000C9446000C94460028
54+
:100040000C9448000C9446000C9446000C94460016
55+
:100050000C9446000C9446000C9446000C94460008
56+
:100060000C9446000C94460011241FBECFEFD8E03C
57+
:10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC
58+
:10008000B207E1F70E9492000C94DC000C9400008F
59+
:100090001F920F920FB60F9211242F933F938F93BD
60+
:1000A0009F93AF93BF938091050190910601A0911A
61+
:1000B0000701B09108013091040123E0230F2D378F
62+
:1000C00020F40196A11DB11D05C026E8230F02965C
63+
:1000D000A11DB11D20930401809305019093060199
64+
:1000E000A0930701B0930801809100019091010154
65+
:1000F000A0910201B09103010196A11DB11D809351
66+
:10010000000190930101A0930201B0930301BF91FC
67+
:10011000AF919F918F913F912F910F900FBE0F90B4
68+
:100120001F901895789484B5826084BD84B58160F1
69+
:1001300084BD85B5826085BD85B5816085BD8091B2
70+
:100140006E00816080936E0010928100809181002A
71+
:100150008260809381008091810081608093810022
72+
:10016000809180008160809380008091B1008460E4
73+
:100170008093B1008091B00081608093B000809145
74+
:100180007A00846080937A0080917A008260809304
75+
:100190007A0080917A00816080937A0080917A0061
76+
:1001A000806880937A001092C100C0E0D0E0209770
77+
:0C01B000F1F30E940000FBCFF894FFCF99
78+
:00000001FF
79+
`
5180
err = buildPath.Join("sketch", "sketch.ino.hex").WriteFile([]byte(fakeSketchHex))
5281
NoError(t, err)
5382

@@ -65,8 +94,8 @@ func TestMergeSketchWithBootloader(t *testing.T) {
6594
NoError(t, err)
6695
mergedSketchHex := string(bytes)
6796

68-
require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n"))
69-
require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n"))
97+
require.Contains(t, mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")
98+
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
7099
}
71100

72101
func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
@@ -88,8 +117,36 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
88117
err := buildPath.Join("sketch").MkdirAll()
89118
NoError(t, err)
90119

91-
fakeSketchHex := "row 1\n" +
92-
"row 2\n"
120+
fakeSketchHex := `:100000000C9434000C9446000C9446000C9446006A
121+
:100010000C9446000C9446000C9446000C94460048
122+
:100020000C9446000C9446000C9446000C94460038
123+
:100030000C9446000C9446000C9446000C94460028
124+
:100040000C9448000C9446000C9446000C94460016
125+
:100050000C9446000C9446000C9446000C94460008
126+
:100060000C9446000C94460011241FBECFEFD8E03C
127+
:10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC
128+
:10008000B207E1F70E9492000C94DC000C9400008F
129+
:100090001F920F920FB60F9211242F933F938F93BD
130+
:1000A0009F93AF93BF938091050190910601A0911A
131+
:1000B0000701B09108013091040123E0230F2D378F
132+
:1000C00020F40196A11DB11D05C026E8230F02965C
133+
:1000D000A11DB11D20930401809305019093060199
134+
:1000E000A0930701B0930801809100019091010154
135+
:1000F000A0910201B09103010196A11DB11D809351
136+
:10010000000190930101A0930201B0930301BF91FC
137+
:10011000AF919F918F913F912F910F900FBE0F90B4
138+
:100120001F901895789484B5826084BD84B58160F1
139+
:1001300084BD85B5826085BD85B5816085BD8091B2
140+
:100140006E00816080936E0010928100809181002A
141+
:100150008260809381008091810081608093810022
142+
:10016000809180008160809380008091B1008460E4
143+
:100170008093B1008091B00081608093B000809145
144+
:100180007A00846080937A0080917A008260809304
145+
:100190007A0080917A00816080937A0080917A0061
146+
:1001A000806880937A001092C100C0E0D0E0209770
147+
:0C01B000F1F30E940000FBCFF894FFCF99
148+
:00000001FF
149+
`
93150
err = buildPath.Join("sketch.ino.hex").WriteFile([]byte(fakeSketchHex))
94151
NoError(t, err)
95152

@@ -107,8 +164,9 @@ func TestMergeSketchWithBootloaderSketchInBuildPath(t *testing.T) {
107164
NoError(t, err)
108165
mergedSketchHex := string(bytes)
109166

110-
require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:107E0000112484B714BE81FFF0D085E080938100F7\n"))
111-
require.True(t, strings.HasSuffix(mergedSketchHex, ":0400000300007E007B\n:00000001FF\n"))
167+
fmt.Println(string(mergedSketchHex))
168+
require.Contains(t, mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")
169+
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
112170
}
113171

114172
func TestMergeSketchWithBootloaderWhenNoBootloaderAvailable(t *testing.T) {
@@ -168,8 +226,36 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) {
168226
err := buildPath.Join("sketch").MkdirAll()
169227
NoError(t, err)
170228

171-
fakeSketchHex := "row 1\n" +
172-
"row 2\n"
229+
fakeSketchHex := `:100000000C9434000C9446000C9446000C9446006A
230+
:100010000C9446000C9446000C9446000C94460048
231+
:100020000C9446000C9446000C9446000C94460038
232+
:100030000C9446000C9446000C9446000C94460028
233+
:100040000C9448000C9446000C9446000C94460016
234+
:100050000C9446000C9446000C9446000C94460008
235+
:100060000C9446000C94460011241FBECFEFD8E03C
236+
:10007000DEBFCDBF21E0A0E0B1E001C01D92A930FC
237+
:10008000B207E1F70E9492000C94DC000C9400008F
238+
:100090001F920F920FB60F9211242F933F938F93BD
239+
:1000A0009F93AF93BF938091050190910601A0911A
240+
:1000B0000701B09108013091040123E0230F2D378F
241+
:1000C00020F40196A11DB11D05C026E8230F02965C
242+
:1000D000A11DB11D20930401809305019093060199
243+
:1000E000A0930701B0930801809100019091010154
244+
:1000F000A0910201B09103010196A11DB11D809351
245+
:10010000000190930101A0930201B0930301BF91FC
246+
:10011000AF919F918F913F912F910F900FBE0F90B4
247+
:100120001F901895789484B5826084BD84B58160F1
248+
:1001300084BD85B5826085BD85B5816085BD8091B2
249+
:100140006E00816080936E0010928100809181002A
250+
:100150008260809381008091810081608093810022
251+
:10016000809180008160809380008091B1008460E4
252+
:100170008093B1008091B00081608093B000809145
253+
:100180007A00846080937A0080917A008260809304
254+
:100190007A0080917A00816080937A0080917A0061
255+
:1001A000806880937A001092C100C0E0D0E0209770
256+
:0C01B000F1F30E940000FBCFF894FFCF99
257+
:00000001FF
258+
`
173259
err = buildPath.Join("sketch", "sketch.ino.hex").WriteFile([]byte(fakeSketchHex))
174260
NoError(t, err)
175261

@@ -187,6 +273,6 @@ func TestMergeSketchWithBootloaderPathIsParameterized(t *testing.T) {
187273
NoError(t, err)
188274
mergedSketchHex := string(bytes)
189275

190-
require.True(t, strings.HasPrefix(mergedSketchHex, "row 1\n:020000023000CC"))
191-
require.True(t, strings.HasSuffix(mergedSketchHex, ":040000033000E000E9\n:00000001FF\n"))
276+
require.Contains(t, mergedSketchHex, ":100000000C9434000C9446000C9446000C9446006A\n")
277+
require.True(t, strings.HasSuffix(mergedSketchHex, ":00000001FF\n"))
192278
}

0 commit comments

Comments
 (0)