From 3be51705d7bdd48312c5303675d2512b50fcbfef Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 31 Oct 2021 09:52:42 -0700 Subject: [PATCH 1/7] Update to SdFat 2.1.0 with UTF-8 support --- libraries/ESP8266SdFat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index 9472c86140..e7ddd0582f 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit 9472c8614033efdaa6064906e7065963581746af +Subproject commit e7ddd0582f857163b81e4d21176997868e957c60 From 4c7f163e6e7c2390b204ad99234f7d42327e8949 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 31 Oct 2021 10:01:28 -0700 Subject: [PATCH 2/7] Update example path, host test paths --- libraries/ESP8266SdFat | 2 +- tests/host/Makefile | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index e7ddd0582f..98ee64c324 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit e7ddd0582f857163b81e4d21176997868e957c60 +Subproject commit 98ee64c324605c66fc99966bdaca12ffe0ca682c diff --git a/tests/host/Makefile b/tests/host/Makefile index 8cb9d89297..cedeb36679 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -109,7 +109,6 @@ CORE_CPP_FILES := \ common/FmtNumber.cpp \ common/FsStructs.cpp \ common/FsDateTime.cpp \ - common/PrintBasic.cpp \ ) \ $(abspath $(LIBRARIES_PATH)/SDFS/src/SDFS.cpp) \ $(abspath $(LIBRARIES_PATH)/SD/src/SD.cpp) \ From 609193f4bc5db6e294e0cc07e8b782766ac5fc23 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 31 Oct 2021 10:20:35 -0700 Subject: [PATCH 3/7] Update host build sources for SdFat to match new --- libraries/ESP8266SdFat | 2 +- tests/common.sh | 1 + tests/host/Makefile | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index 98ee64c324..64dc38de20 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit 98ee64c324605c66fc99966bdaca12ffe0ca682c +Subproject commit 64dc38de2089ce31210273a15c9d8057c95780b4 diff --git a/tests/common.sh b/tests/common.sh index a99ff6a9c0..537a87a315 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -21,6 +21,7 @@ function skip_ino() /TeensySdioDemo/ /UserChipSelectFunction/ /UserSPIDriver/ +/examples.donotuse/ EOL echo $ino | grep -q -F "$skiplist" echo $(( 1 - $? )) diff --git a/tests/host/Makefile b/tests/host/Makefile index cedeb36679..b27aef93c6 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -104,9 +104,11 @@ CORE_CPP_FILES := \ FatLib/FatFilePrint.cpp \ FatLib/FatFileSFN.cpp \ FatLib/FatFormatter.cpp \ + FatLib/FatName.cpp \ FatLib/FatVolume.cpp \ FatLib/FatPartition.cpp \ common/FmtNumber.cpp \ + common/FsCache.cpp \ common/FsStructs.cpp \ common/FsDateTime.cpp \ ) \ From 2b9e259bfd02c9d217f46c4a9e236b856d8b2ab5 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Wed, 3 Nov 2021 00:52:31 -0700 Subject: [PATCH 4/7] Upgrade to SdFat 2.1.1 from scratch, add UTF-8 support Use latest upstream SdFat library to avoid need for manual patched namespaces around the core. Include UTF-8 support as well. --- libraries/ESP8266SdFat | 2 +- libraries/SD/src/SD.h | 17 +++------ libraries/SDFS/src/SDFS.cpp | 26 ++++++------- libraries/SDFS/src/SDFS.h | 74 ++++++++++++++++++------------------- tests/common.sh | 16 ++++++-- tests/host/Makefile | 3 ++ 6 files changed, 71 insertions(+), 67 deletions(-) diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index 64dc38de20..4563f80d9d 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit 64dc38de2089ce31210273a15c9d8057c95780b4 +Subproject commit 4563f80d9d47ae99c3d428edd55414ed2a8b6e61 diff --git a/libraries/SD/src/SD.h b/libraries/SD/src/SD.h index eb9372b2a8..cfb7872b26 100644 --- a/libraries/SD/src/SD.h +++ b/libraries/SD/src/SD.h @@ -25,9 +25,9 @@ #include #undef FILE_READ -#define FILE_READ sdfat::O_READ +#define FILE_READ ((uint8_t)O_READ) #undef FILE_WRITE -#define FILE_WRITE (sdfat::O_READ | sdfat::O_WRITE | sdfat::O_CREAT | sdfat::O_APPEND) +#define FILE_WRITE ((uint8_t)(O_READ | O_WRITE | O_CREAT | O_APPEND)) class SDClass { @@ -159,9 +159,9 @@ class SDClass { private: const char *getMode(uint8_t mode) { - bool read = (mode & sdfat::O_READ) ? true : false; - bool write = (mode & sdfat::O_WRITE) ? true : false; - bool append = (mode & sdfat::O_APPEND) ? true : false; + bool read = (mode & O_READ) ? true : false; + bool write = (mode & O_WRITE) ? true : false; + bool append = (mode & O_APPEND) ? true : false; if ( read & !write ) { return "r"; } else if ( !read & write & !append ) { return "w+"; } else if ( !read & write & append ) { return "a"; } @@ -183,10 +183,6 @@ class SDClass { }; -// Expose FatStructs.h helpers for MS-DOS date/time for use with dateTimeCallback -static inline uint16_t FAT_DATE(uint16_t year, uint8_t month, uint8_t day) { - return (year - 1980) << 9 | month << 5 | day; -} static inline uint16_t FAT_YEAR(uint16_t fatDate) { return 1980 + (fatDate >> 9); } @@ -196,9 +192,6 @@ static inline uint8_t FAT_MONTH(uint16_t fatDate) { static inline uint8_t FAT_DAY(uint16_t fatDate) { return fatDate & 0X1F; } -static inline uint16_t FAT_TIME(uint8_t hour, uint8_t minute, uint8_t second) { - return hour << 11 | minute << 5 | second >> 1; -} static inline uint8_t FAT_HOUR(uint16_t fatTime) { return fatTime >> 11; } diff --git a/libraries/SDFS/src/SDFS.cpp b/libraries/SDFS/src/SDFS.cpp index e7a891a08a..5725a6ae08 100644 --- a/libraries/SDFS/src/SDFS.cpp +++ b/libraries/SDFS/src/SDFS.cpp @@ -64,13 +64,13 @@ FileImplPtr SDFSImpl::open(const char* path, OpenMode openMode, AccessMode acces } free(pathStr); } - sdfat::File32 fd = _fs.open(path, flags); + File32 fd = _fs.open(path, flags); if (!fd) { DEBUGV("SDFSImpl::openFile: fd=%p path=`%s` openMode=%d accessMode=%d", &fd, path, openMode, accessMode); return FileImplPtr(); } - auto sharedFd = std::make_shared(fd); + auto sharedFd = std::make_shared(fd); return std::make_shared(this, sharedFd, path); } @@ -90,14 +90,14 @@ DirImplPtr SDFSImpl::openDir(const char* path) } // At this point we have a name of "/blah/blah/blah" or "blah" or "" // If that references a directory, just open it and we're done. - sdfat::File32 dirFile; + File32 dirFile; const char *filter = ""; if (!pathStr[0]) { // openDir("") === openDir("/") - dirFile = _fs.open("/", sdfat::O_RDONLY); + dirFile = _fs.open("/", O_RDONLY); filter = ""; } else if (_fs.exists(pathStr)) { - dirFile = _fs.open(pathStr, sdfat::O_RDONLY); + dirFile = _fs.open(pathStr, O_RDONLY); if (dirFile.isDir()) { // Easy peasy, path specifies an existing dir! filter = ""; @@ -107,12 +107,12 @@ DirImplPtr SDFSImpl::openDir(const char* path) char *ptr = strrchr(pathStr, '/'); if (!ptr) { // No slashes, open the root dir - dirFile = _fs.open("/", sdfat::O_RDONLY); + dirFile = _fs.open("/", O_RDONLY); filter = pathStr; } else { // We've got slashes, open the dir one up *ptr = 0; // Remove slash, truncare string - dirFile = _fs.open(pathStr, sdfat::O_RDONLY); + dirFile = _fs.open(pathStr, O_RDONLY); filter = ptr + 1; } } @@ -122,12 +122,12 @@ DirImplPtr SDFSImpl::openDir(const char* path) char *ptr = strrchr(pathStr, '/'); if (!ptr) { // No slashes, open the root dir - dirFile = _fs.open("/", sdfat::O_RDONLY); + dirFile = _fs.open("/", O_RDONLY); filter = pathStr; } else { // We've got slashes, open the dir one up *ptr = 0; // Remove slash, truncare string - dirFile = _fs.open(pathStr, sdfat::O_RDONLY); + dirFile = _fs.open(pathStr, O_RDONLY); filter = ptr + 1; } } @@ -135,7 +135,7 @@ DirImplPtr SDFSImpl::openDir(const char* path) DEBUGV("SDFSImpl::openDir: path=`%s`\n", path); return DirImplPtr(); } - auto sharedDir = std::make_shared(dirFile); + auto sharedDir = std::make_shared(dirFile); auto ret = std::make_shared(filter, this, sharedDir, pathStr); free(pathStr); return ret; @@ -145,12 +145,12 @@ bool SDFSImpl::format() { if (_mounted) { return false; } - sdfat::SdCardFactory cardFactory; - sdfat::SdCard* card = cardFactory.newCard(sdfat::SdSpiConfig(_cfg._csPin, DEDICATED_SPI, _cfg._spiSettings)); + SdCardFactory cardFactory; + SdCard* card = cardFactory.newCard(SdSpiConfig(_cfg._csPin, DEDICATED_SPI, _cfg._spiSettings)); if (!card || card->errorCode()) { return false; } - sdfat::FatFormatter fatFormatter; + FatFormatter fatFormatter; uint8_t *sectorBuffer = new uint8_t[512]; bool ret = fatFormatter.format(card, sectorBuffer, nullptr); delete[] sectorBuffer; diff --git a/libraries/SDFS/src/SDFS.h b/libraries/SDFS/src/SDFS.h index 2bc1dc972c..d2b3725574 100644 --- a/libraries/SDFS/src/SDFS.h +++ b/libraries/SDFS/src/SDFS.h @@ -97,11 +97,11 @@ class SDFSImpl : public FSImpl return false; } info.maxOpenFiles = 999; // TODO - not valid - info.blockSize = _fs.vol()->sectorsPerCluster() * _fs.vol()->bytesPerSector(); + info.blockSize = _fs.vol()->bytesPerCluster(); info.pageSize = 0; // TODO ? info.maxPathLength = 255; // TODO ? info.totalBytes =_fs.vol()->clusterCount() * info.blockSize; - info.usedBytes = info.totalBytes - (_fs.vol()->freeClusterCount() * _fs.vol()->sectorsPerCluster() * _fs.vol()->bytesPerSector()); + info.usedBytes = info.totalBytes - (_fs.vol()->freeClusterCount() * _fs.vol()->bytesPerCluster()); return true; } @@ -156,7 +156,7 @@ class SDFSImpl : public FSImpl format(); _mounted = _fs.begin(_cfg._csPin, _cfg._spiSettings); } - sdfat::FsDateTime::setCallback(dateTimeCB); + FsDateTime::setCallback(dateTimeCB); return _mounted; } @@ -185,7 +185,7 @@ class SDFSImpl : public FSImpl return (totalClusters() / blocksPerCluster()); } size_t clusterSize() { - return blocksPerCluster() * _fs.vol()->bytesPerSector(); + return _fs.vol()->bytesPerCluster(); } size_t size() { return (clusterSize() * totalClusters()); @@ -229,8 +229,7 @@ class SDFSImpl : public FSImpl friend class SDFileImpl; friend class SDFSDirImpl; - sdfat::SdFat* getFs() - { + SdFat* getFs() { return &_fs; } @@ -238,24 +237,25 @@ class SDFSImpl : public FSImpl static uint8_t _getFlags(OpenMode openMode, AccessMode accessMode) { uint8_t mode = 0; if (openMode & OM_CREATE) { - mode |= sdfat::O_CREAT; + mode |= O_CREAT; } if (openMode & OM_APPEND) { - mode |= sdfat::O_AT_END; + mode |= O_AT_END; } if (openMode & OM_TRUNCATE) { - mode |= sdfat::O_TRUNC; - } - if (accessMode & AM_READ) { - mode |= sdfat::O_READ; + mode |= O_TRUNC; } - if (accessMode & AM_WRITE) { - mode |= sdfat::O_WRITE; + if ((accessMode & (AM_READ | AM_WRITE)) == (AM_READ | AM_WRITE)) { + mode |= O_RDWR; + } else if (accessMode & AM_READ) { + mode |= O_READ; + } else if (accessMode & AM_WRITE) { + mode |= O_WRITE; } return mode; } - sdfat::SdFat _fs; + SdFat _fs; SDFSConfig _cfg; bool _mounted; }; @@ -264,7 +264,7 @@ class SDFSImpl : public FSImpl class SDFSFileImpl : public FileImpl { public: - SDFSFileImpl(SDFSImpl *fs, std::shared_ptr fd, const char *name) + SDFSFileImpl(SDFSImpl *fs, std::shared_ptr fd, const char *name) : _fs(fs), _fd(fd), _opened(true) { _name = std::shared_ptr(new char[strlen(name) + 1], std::default_delete()); @@ -380,7 +380,7 @@ class SDFSFileImpl : public FileImpl time_t getLastWrite() override { time_t ftime = 0; if (_opened && _fd) { - sdfat::DirFat_t tmp; + DirFat_t tmp; if (_fd.get()->dirEntry(&tmp)) { ftime = SDFSImpl::FatToTimeT(*(uint16_t*)tmp.modifyDate, *(uint16_t*)tmp.modifyTime); } @@ -391,7 +391,7 @@ class SDFSFileImpl : public FileImpl time_t getCreationTime() override { time_t ftime = 0; if (_opened && _fd) { - sdfat::DirFat_t tmp; + DirFat_t tmp; if (_fd.get()->dirEntry(&tmp)) { ftime = SDFSImpl::FatToTimeT(*(uint16_t*)tmp.createDate, *(uint16_t*)tmp.createTime); } @@ -400,16 +400,16 @@ class SDFSFileImpl : public FileImpl } protected: - SDFSImpl* _fs; - std::shared_ptr _fd; - std::shared_ptr _name; - bool _opened; + SDFSImpl* _fs; + std::shared_ptr _fd; + std::shared_ptr _name; + bool _opened; }; class SDFSDirImpl : public DirImpl { public: - SDFSDirImpl(const String& pattern, SDFSImpl* fs, std::shared_ptr dir, const char *dirPath = nullptr) + SDFSDirImpl(const String& pattern, SDFSImpl* fs, std::shared_ptr dir, const char *dirPath = nullptr) : _pattern(pattern), _fs(fs), _dir(dir), _valid(false), _dirPath(nullptr) { if (dirPath) { @@ -484,14 +484,14 @@ class SDFSDirImpl : public DirImpl { const int n = _pattern.length(); do { - sdfat::File32 file; - file.openNext(_dir.get(), sdfat::O_READ); + File32 file; + file.openNext(_dir.get(), O_READ); if (file) { _valid = 1; _size = file.fileSize(); _isFile = file.isFile(); _isDirectory = file.isDir(); - sdfat::DirFat_t tmp; + DirFat_t tmp; if (file.dirEntry(&tmp)) { _time = SDFSImpl::FatToTimeT(*(uint16_t*)tmp.modifyDate, *(uint16_t*)tmp.modifyTime); _creation = SDFSImpl::FatToTimeT(*(uint16_t*)tmp.createDate, *(uint16_t*)tmp.createTime); @@ -516,17 +516,17 @@ class SDFSDirImpl : public DirImpl } protected: - String _pattern; - SDFSImpl* _fs; - std::shared_ptr _dir; - bool _valid; - char _lfn[64]; - time_t _time; - time_t _creation; - std::shared_ptr _dirPath; - uint32_t _size; - bool _isFile; - bool _isDirectory; + String _pattern; + SDFSImpl* _fs; + std::shared_ptr _dir; + bool _valid; + char _lfn[64]; + time_t _time; + time_t _creation; + std::shared_ptr _dirPath; + uint32_t _size; + bool _isFile; + bool _isDirectory; }; }; // namespace sdfs diff --git a/tests/common.sh b/tests/common.sh index 537a87a315..00eede57b8 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -10,18 +10,26 @@ function skip_ino() /#attic/ /AvrAdcLogger/ /BackwardCompatibility/ +/bench/ +/BufferedPrint/ +/DirectoryFunctions/ /examplesV1/ -/ExFatFormatter/ /ExFatLogger/ -/ExFatUnicodeTest/ +/MinimumSizeSdReader/ +/OpenNext/ +/QuickStart/ +/ReadCsvFile/ +/rename/ /RtcTimestampTest/ +/SdErrorCodes/ /SoftwareSpi/ -/STM32Test/ +/TeensyDmaAdcLogger/ /TeensyRtcTimestamp/ /TeensySdioDemo/ +/TeensySdioLogger/ +/UnicodeFilenames/ /UserChipSelectFunction/ /UserSPIDriver/ -/examples.donotuse/ EOL echo $ino | grep -q -F "$skiplist" echo $(( 1 - $? )) diff --git a/tests/host/Makefile b/tests/host/Makefile index b27aef93c6..a2353a0841 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -111,6 +111,9 @@ CORE_CPP_FILES := \ common/FsCache.cpp \ common/FsStructs.cpp \ common/FsDateTime.cpp \ + common/FsUtf.cpp \ + common/FsName.cpp \ + common/upcase.cpp \ ) \ $(abspath $(LIBRARIES_PATH)/SDFS/src/SDFS.cpp) \ $(abspath $(LIBRARIES_PATH)/SD/src/SD.cpp) \ From 88e20f5c2acff389b88696005b5b752e4d670f05 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Wed, 3 Nov 2021 01:46:15 -0700 Subject: [PATCH 5/7] Fix uninitted variable error in SdFat --- libraries/ESP8266SdFat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index 4563f80d9d..9598a59a25 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit 4563f80d9d47ae99c3d428edd55414ed2a8b6e61 +Subproject commit 9598a59a259d7bdd914e34d762c7695bfb4e79a5 From 1902618910bfe0d50fb620522d21e22035bddfe5 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Wed, 3 Nov 2021 02:24:09 -0700 Subject: [PATCH 6/7] Allow many ESP8266SdFat examples to build properly Add the new examples back to CI runs for more coverage --- libraries/ESP8266SdFat | 2 +- tests/common.sh | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index 9598a59a25..6b05a18934 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit 9598a59a259d7bdd914e34d762c7695bfb4e79a5 +Subproject commit 6b05a189342238c6dfa737c66f0d50415373e37b diff --git a/tests/common.sh b/tests/common.sh index 00eede57b8..b6ce2fe61e 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -9,25 +9,13 @@ function skip_ino() read -d '' skiplist << EOL || true /#attic/ /AvrAdcLogger/ -/BackwardCompatibility/ -/bench/ -/BufferedPrint/ -/DirectoryFunctions/ /examplesV1/ -/ExFatLogger/ -/MinimumSizeSdReader/ -/OpenNext/ -/QuickStart/ -/ReadCsvFile/ -/rename/ /RtcTimestampTest/ -/SdErrorCodes/ /SoftwareSpi/ /TeensyDmaAdcLogger/ /TeensyRtcTimestamp/ /TeensySdioDemo/ /TeensySdioLogger/ -/UnicodeFilenames/ /UserChipSelectFunction/ /UserSPIDriver/ EOL From 06d430c5e059448c776e2d1577b1a18bf6faec55 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Thu, 4 Nov 2021 13:00:15 -0700 Subject: [PATCH 7/7] Update to 115200 for SD examples --- libraries/ESP8266SdFat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index 6b05a18934..3a6b688b1b 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit 6b05a189342238c6dfa737c66f0d50415373e37b +Subproject commit 3a6b688b1b5112d149295813660ef9b2b7cc0ee5