From: Steinar H. Gunderson Date: Sat, 1 Dec 2018 23:11:12 +0000 (+0100) Subject: Merge remote-tracking branch 'futatabi/master' X-Git-Tag: 1.8.0~76 X-Git-Url: https://git.sesse.net/?a=commitdiff_plain;h=9b7d691b4cc5db7dbfc18c82e86c1207fcac4722;hp=6e116a6bbeb2c047a3bfb084395ec601ce211e6c;p=nageru Merge remote-tracking branch 'futatabi/master' This merges Nageru and Futatabi, since they are fairly closely related and also share a fair amount of code. --- diff --git a/.gitignore b/.gitignore index 51f813a..c0b5588 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,2 @@ -*.mp4 -*.png -*.flo -*.pgm -*.ppm -*.sw* -.ycm_extra_conf.py obj/ -frames/ -futatabi.db +.ycm_extra_conf.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..5a47877 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "bmusb"] + path = bmusb + url = http://git.sesse.net/bmusb diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..309c9d8 --- /dev/null +++ b/NEWS @@ -0,0 +1,386 @@ +Nageru 1.7.5, November 11th, 2018 + + - Fix a bug where --record-x264-video would not work when VA-API was + not present, making the option rather useless (broken in 1.7.2). + Bug reported by Peter De Schrijver. + + - The build system has been switched to Meson; see the README for new + build instructions. + + - Various smaller fixes. + + +Nageru 1.7.4, August 31st, 2018 + + - Rework the x264 speedcontrol presets, again. (They earlier assumed + we could control B-frame settings on the fly, which we cannot with + threaded lookahead.) Also support x264 >= 153, which can support + multiple bit depths in the same library. + + - Default to SDI inputs instead of HDMI. + + - Add a mode to run in full screen (--fullscreen). Adapted from a patch + by Yoann Dubreuil. + + - Add support for lift/gamma/gain in the theme. Patch by Alexandre Thomazo. + + +Nageru 1.7.3, May 22nd, 2018 + + - When using multichannel audio, add a control for adjusting the + stereo width (from normal stereo to mono, all the way to + inverted stereo). + + - Removed --http-coarse-timebase (it is now always on). + + - Various bugfixes. + + +Nageru 1.7.2, April 28th, 2018 + + - Several improvements to video (FFmpeg) inputs: You can now use + them as audio sources, you can right-click on video channels + to change URL/filename on-the-fly, themes can ask for forced + disconnection (useful for network sources that are hanging), + and various other improvements. Be aware that the audio support + may still be somewhat rough, as A/V sync of arbitrary video + playout is a hard problem. + + - The included themes have been fixed to properly make the returned + chain preparation functions independent of global state (e.g. if + the white balance for a channel was changed before the frame was + actually rendered). If you are using a custom theme, you may want + to apply similar fixes to it. + + - In Metacube stream output, mark each keyframe with a pts metadata + block. This allows Cubemap 1.4.0 or newer to serve fMP4 fragments + for HLS from Nageru's output, without any further remuxing or + transcoding. + + - If needed, Nageru will now automatically try to autodetect a + usable --va-display parameter by probing all DRM nodes for H.264 + encoders. This removes the need to set --va-display in almost all + cases, and also removes the dependency on libpci. + + - For GPUs that support querying available memory (in practice only + NVIDIA GPUs at the current time), expose the amount of used/total + GPU memory both on standard output and in the Prometheus metrics + (as well as included Grafana dashboard). + + - The Grafana dashboard now supports heatmaps for the chosen x264 + speedcontrol preset (requires Grafana 5.1 or newer). (There used to + be a heatmap earlier, but it was all broken.) + + - Various bugfixes. + + +Nageru 1.7.1, March 26th, 2018 + + - Various bugfixes, mostly related to HTML and video inputs. + + +Nageru 1.7.0, March 8th, 2018 + + - Support for HTML5 graphics directly in Nageru, through CEF + (Chromium Embedded Framework). This performs better and is more + flexible than integrating with CasparCG over a socket. Note that + CEF is an optional component; see the documentation for more + information. + + - Add an HTTP endpoint for enumerating channels and one for getting + only their colors. Intended for remote tally applications; + set the documentation. + + - Add a video grid display that removes the audio controls and shows + the video channels only, potentially in multiple rows if that makes + for a larger viewing area. + + - Themes can now present simple menus in the Nageru UI. See the + documentation for more information. + + - Various bugfixes. + + +Nageru 1.6.4, January 25th, 2018 + + - Fix compilation with the upcoming FFmpeg 3.5. + + - Switch to LuaJIT for the theme engine, which is faster. + + - Various bugfixes and smaller optimizations. + + +Nageru 1.6.3, November 8th, 2017 + + - Add quick-cut keys (Q, W, E, etc.) below the preview keys. + Since it's easy to hit these by accident and put up a signal + you didn't want, they are disabled by default (they can be + enabled in the video menu, or with the command line flag + --quick-cut-keys). + + - Rework the x264 speedcontrol presets to better match newer + x264 versions. + + - Add an option for changing the HTTP port (--http-port). + + - Various smaller bug and integration fixes. + + +Nageru 1.6.2, July 16th, 2017 + + - Various smaller Kaeru fixes, mostly around metrics. Also, + you can now adjust the x264 bitrate in Kaeru (in 100 kbit/sec + increments) by sending SIGUSR1 (higher) or SIGUSR2 (lower). + + +Nageru 1.6.1, July 9th, 2017 + + - Add native export of Prometheus metrics. + + - Rework the frame queue drop algorithm. The new one should handle tricky + situations much better, especially when a card is drifting very slowly + against the master timer. + + - Add Kaeru, an experimental transcoding tool based on Nageru code. + Kaeru can run headless on a server without a GPU to transcode a + Nageru stream into a lower-bitrate one, replacing VLC. + + - Work around a bug in some versions of NVIDIA's OpenGL drivers that would + crash Nageru after about three hours (fix in cooperation with Movit). + + - Fix a crash with i965-va-driver 1.8.x. + + - Reduce mutex contention in certain critical places, causing lower tail + latency in the mixer. + + +Nageru 1.6.0, May 29th, 2017 + + - Add support for having videos (from file or from URL) as a separate + input channels, albeit with some limitations. Apart from the obvious use of + looping pause clips or similar, this can be used to integrate with CasparCG; + see the manual for more details. + + - Add a frame analyzer (accessible from the Video menu) containing an + RGB histogram and a color dropped tool. This is useful in calibrating + video chains by playing back a known signal. Note that this adds a + dependency on QCustomPlot. + + - Allow overriding Y'CbCr input interpretation, for inputs that don't + use the correct settings. Also, Rec. 601 is now used by default instead + of Rec. 709 for SD resolutions. + + - Support other sample rates than 48000 Hz from bmusb. + + +Nageru 1.5.0, April 5th, 2017 + + - Support for low-latency HDMI/SDI output in addition to (or instead of) the + stream. This currently only works with DeckLink cards, not bmusb. See the + manual for more information. + + - Support changing the resolution from the command line, instead of locking + everything to 1280x720. + + - The A/V sync code has been rewritten to be more in line with Fons + Adriaensen's original paper. It handles several cases much better, + in particular when trying to match 59.94 and 60 Hz sources to each other. + However, it might occasionally need a few extra seconds on startup to + lock properly if startup is slow. + + - Add support for using x264 for the disk recording. This makes it possible, + among other things, to run Nageru on a machine entirely without VA-API + support. + + - Support for 10-bit Y'CbCr, both on input and output. (Output requires + x264 disk recording, as Quick Sync Video does not support 10-bit H.264.) + This requires compute shader support, and is in general a little bit + slower on input and output, due to the extra amount of data being shuffled + around. Intermediate precision is 16-bit floating-point or better, + as before. + + - Enable input mode autodetection for DeckLink cards that support it. + (bmusb mode has always been autodetected.) + + - Add functionality to add a time code to the stream; useful for debugging + latency. + + - The live display is now both more performant and of higher image quality. + + - Fix a long-standing issue where the preview displays would be too bright + when using an NVIDIA GPU. (This did not affect the finished stream.) + + - Many other bugfixes and small improvements. + + +Nageru 1.4.2, November 24th, 2016 + + - Fix a thread race that would sometimes cause x264 streaming to go awry. + + +Nageru 1.4.1, November 6th, 2016 + + - Various bugfixes. + + +Nageru 1.4.0, October 26th, 2016 + + - Support for multichannel (or more accurately, multi-bus) audio, + choosable from the UI or using the --multichannel command-line + flag. In multichannel mode, you can take in inputs from multiple + different sources (or different channels on the same source, for + multichannel sound cards), apply effects to them separately and then + mix them together. This includes both audio from the video cards + as well as ALSA inputs, including hotplug. Ola Gundelsby contributed + invaluable feedback on this feature throughout the entire + development cycle. + + - Support for having MIDI controllers control various aspects of the + audio UI, with relatively flexible mapping. Note that different + MIDI controllers can vary significantly in what protocol they speak, + so Nageru will not necessarily work with all. (The primary testing + controller has been the Akai MIDImix, and a pre-made mapping for + that is included. The Korg nanoKONTROL2 has also been tested and + works, but it requires some Korg-specific SysEx commands to make + the buttons and lights work.) + + - Add a disk space indicator to the main window. + + - Various bugfixes. In particular, an issue where the audio would pitch + up sharply after a series of many dropped frames has been fixed. + + +Nageru 1.3.4, August 2nd, 2016 + + - Various bugfixes. + + +Nageru 1.3.3, July 27th, 2016 + + - Various changes to make distribution packaging easier; in particular, + theme data can be picked up from /usr/local/share/nageru. + + - Fix various FFmpeg deprecation warnings, now that we need FFmpeg + 3.1 for other reasons anyway. + + +Nageru 1.3.2, July 23rd, 2016 + + - Allow limited hotplugging (unplugging and replugging) of USB cards. + You can use the new command-line option --num-fake-cards (-C) to add + fake cards that show only a single color and that will be replaced + by real cards as you plug them in; you can also unplug cards and have + them be replaced by fake cards. Fake cards can also be used for testing + Nageru without actually having any video cards available. + + - Add Metacube timestamping of every keyframe, for easier detection of + streams not keeping up. Works with the new timestamp feature of + Cubemap 1.3.1. Will be ignored (save for some logging) in older + Cubemap versions. + + - The included default theme has been reworked and cleaned up to be + more understandable and extensible. + + - Add more command-line options for initial audio setup. + + +Nageru 1.3.1, July 1st, 2016 + + - Various display bugfixes. + + +Nageru 1.3.0, June 26th, 2016 + + - It is now possible, given enough CPU power (e.g., a quad-core Haswell or + faster desktop CPU), to output a stream that is suitable for streaming + directly to end users without further transcoding. In particular, this + includes support for encoding the network stream with x264 (the stream + saved to disk is still done using Quick Sync), for Metacube framing (for + streaming to the Cubemap reflector), and for choosing the network stream + mux. For more information, see the README. + + - Add a flag (--disable-alsa-output) to disable ALSA monitoring output. + + - Do texture uploads from the main thread instead of from separate threads; + may or may not improve stability with NVIDIA's proprietary drivers. + + - When beginning a new video segment, the shutdown of the old encoder + is now done in a background thread, in order to not disturb the external + stream. The audio still goes into a somewhat random stream, though. + + - You can now override the default stream-to-card mapping with --map-signal= + on the command line. + + - Nageru now tries to lock itself into RAM if it has the permissions to do + so, for better realtime behavior. (Writing the stream to disk tends to + fill the buffer cache, eventually paging less-used parts of Nageru out.) + + - Various fixes for deadlocks, memory leaks, and many other errors. + + +Nageru 1.2.1, April 15th, 2016 + + - Images are now updated from disk about every second, so that it is possible + to update e.g. overlays during streaming, although somewhat slowly. + + - Fix support for PNG images. + + - You can now send SIGHUP to start a new cut instead of using the menu. + + - Added a --help option. + + - Various tweaks to OpenGL fence handling. + + +Nageru 1.2.0, April 6th, 2016 + + - Support for Blackmagic's PCI and Thunderbolt cards, using the official + (closed-source) Blackmagic drivers. (You do not need the SDK installed, though.) + You can use PCI and USB cards pretty much interchangeably. + + - Much more stable handling of frame queues on non-master cards. In particular, + you can have a master card on 50 Hz and another card on 60 Hz without getting + lots of warning messages and a 10+ frame latency on the second card. + + - Many new options in the right click menu on cards: Adjustable video inputs, + adjustable audio inputs, adjustable resolutions, ability to select card for + master clock. + + - Add support for starting with almost all audio processing turned off + (--flat-audio). + + - The UI now marks inputs with red or green to mark them as participating in + the live or preview signal, respectively. Red takes priority. (Actually, + it merely asks the theme for a color for each input; the theme contains + the logic.) + + - Add support for uncompressed video instead of H.264 on the HTTP server, + while still storing H.264 to files (--http-uncompressed-video). Note that + depending on your client, this might not actually be more CPU efficient + even on localhost, so be sure to check. + + - Add a simpler, less featureful theme (simple.lua) that should be easier to + understand for beginners. Themes are now also choosable with -t on the command + line. + + - Too many bugfixes and small tweaks to list. In particular, many memory leaks + in the streaming part have been identified and fixed. + + +Nageru 1.1.0, February 24th, 2016 + + - Support doing the H.264 encoding on a different graphics device from the one + doing the mixing. In particular, this makes it possible to use Nageru on an + NVIDIA GPU while still encoding H.264 video using Intel Quick Sync (NVENC + is not supported yet) -- it is less efficient since the data needs to be read + back via the CPU, but the NVIDIA cards and drivers are so much faster that it + doesn't really matter. Tested on a GTX 950 with the proprietary drivers. + + - In the included example theme, fix fading to/from deinterlaced sources. + + - Various smaller compilation, distribution and documentation fixes. + + +Nageru 1.0.0, January 30th, 2016 + + - Initial release. diff --git a/Nageru-Grafana.json b/Nageru-Grafana.json new file mode 100644 index 0000000..013e738 --- /dev/null +++ b/Nageru-Grafana.json @@ -0,0 +1,1741 @@ +{ + "__inputs": [ + { + "name": "DS_EXAMPLE", + "label": "Data source", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.1.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1524926525808, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 63, + "panels": [], + "repeat": "instance", + "title": "$instance", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_EXAMPLE}", + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 37, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time() - nageru_start_time_seconds{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Nageru uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_EXAMPLE}", + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "instance", + "targets": [ + { + "expr": "nageru_disk_free_bytes / ignoring(destination) deriv(nageru_mux_written_bytes{destination=\"files_total\",instance=~\"$instance\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Disk space remaining", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_EXAMPLE}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "nageru_num_connected_clients{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Connected clients", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_EXAMPLE}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 46, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(nageru_input_has_signal_bool{cardtype=\"ffmpeg\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Connected FFmpeg inputs", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_EXAMPLE}", + "decimals": 1, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " LU", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "nageru_audio_loudness_integrated_lufs{instance=~\"$instance\"} + 23", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Overall audio level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "nageru_audio_correlation{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Audio correlation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": "1", + "min": "-1", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 64, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(nageru_decklink_output_completed_frames{status!=\"completed\",instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Output frames {{status}}", + "refId": "A", + "step": 20 + }, + { + "expr": "sum(rate(nageru_input_dropped_frames_error{instance=~\"$instance\"}[1m])) without (cardtype)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Error frames card {{card}}", + "refId": "B", + "step": 20 + }, + { + "expr": "rate(nageru_quick_sync_stalled_frames{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Quick Sync stalled frames", + "refId": "C", + "step": 20 + }, + { + "expr": "rate(nageru_x264_dropped_frames{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "x264 dropped frames", + "refId": "D", + "step": 20 + }, + { + "expr": "rate(nageru_x264_speedcontrol_idle_frames{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "x264 speedcontrol idle frames", + "refId": "E", + "step": 20 + }, + { + "expr": "rate(nageru_x264_speedcontrol_late_frames{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "x264 speedcontrol late frames", + "refId": "F", + "step": 20 + }, + { + "expr": "rate(nageru_memory_gpu_evictions{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "GPU memory evictions", + "refId": "G" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Potential performance problems", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nageru_input_queue_duped_frames{instance=~\"$instance\"}[1m])) without (cardtype)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Duplicated frames (queue starvation) card {{card}}", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "expr": "sum(rate(nageru_input_dropped_frames_jitter{instance=~\"$instance\"}[1m])) without (cardtype)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Dropped frames card {{card}}", + "refId": "B", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Queue events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 65, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "8 * rate(nageru_mux_stream_bytes{destination=\"http\",instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{stream}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Stream bitrates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "8 * rate(nageru_mux_stream_bytes{destination=\"current_file\",instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{stream}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk bitrates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 66, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "90-percentile", + "fillBelowTo": "10-percentile", + "lines": false + }, + { + "alias": "10-percentile", + "lines": false + }, + { + "alias": "Average", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.9, rate(nageru_x264_crf_bucket{instance=~\"$instance\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90-percentile", + "refId": "B", + "step": 30 + }, + { + "expr": "histogram_quantile(0.1, rate(nageru_x264_crf_bucket{instance=~\"$instance\"}[1m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "10-percentile", + "refId": "A", + "step": 30 + }, + { + "expr": "rate(nageru_x264_crf_sum{instance=~\"$instance\"}[1m]) / rate(nageru_x264_crf_count{instance=~\"$instance\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "x264 average CRF (lower is better)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": 0, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_EXAMPLE}", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 29 + }, + "heatmap": {}, + "highlightCards": true, + "id": 55, + "legend": { + "show": false + }, + "links": [], + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "expr": "rate(nageru_x264_speedcontrol_preset_used_frames_bucket{instance=~\"$instance\"}[1m])", + "format": "heatmap", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{le}}", + "refId": "A", + "step": 30 + } + ], + "title": "x264 speed control chosen preset", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": "", + "yAxis": { + "decimals": null, + "format": "none", + "logBase": 1, + "max": "26", + "min": "0", + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": 1 + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 36 + }, + "id": 62, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "nageru_x264_speedcontrol_buffer_available_seconds{instance=~\"$instance\"} / nageru_x264_speedcontrol_buffer_size_seconds{instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "x264 speed control buffer available", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "x264 buffer space available", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 67, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "card", + "seriesOverrides": [ + { + "alias": "90-percentile", + "fillBelowTo": "10-percentile", + "lines": false + }, + { + "alias": "10-percentile", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "nageru_latency_seconds{measuring_point=\"decklink_output\",frame_type=\"total\",quantile=\"0.9\",frame_age=\"0\",card=\"$card\",instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "90-percentile", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "nageru_latency_seconds{measuring_point=\"decklink_output\",frame_type=\"total\",quantile=\"0.1\",frame_age=\"0\",card=\"$card\",instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "10-percentile", + "refId": "B", + "step": 30 + }, + { + "expr": "sum(\n rate(nageru_latency_seconds_sum{measuring_point=\"decklink_output\",frame_age=\"0\",card=\"$card\",instance=~\"$instance\"}[1m])\n) without (frame_type) / sum(\n rate(nageru_latency_seconds_count{measuring_point=\"decklink_output\",frame_age=\"0\",card=\"$card\",instance=~\"$instance\"}[1m])\n) without (frame_type)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "HDMI/SDI latency, card $card", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 20, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "card", + "seriesOverrides": [ + { + "alias": "90-percentile", + "fillBelowTo": "10-percentile", + "lines": false + }, + { + "alias": "10-percentile", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "nageru_latency_seconds{measuring_point=\"mixer\",frame_type=\"total\",quantile=\"0.9\",frame_age=\"0\",card=\"$card\",instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "90-percentile", + "metric": "", + "refId": "A", + "step": 30 + }, + { + "expr": "nageru_latency_seconds{measuring_point=\"mixer\",frame_type=\"total\",quantile=\"0.1\",frame_age=\"0\",card=\"$card\",instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "10-percentile", + "refId": "B", + "step": 30 + }, + { + "expr": "sum(\n rate(nageru_latency_seconds_sum{measuring_point=\"mixer\",frame_age=\"0\",card=\"$card\",instance=~\"$instance\"}[1m])\n) without (frame_type) / sum(\n rate(nageru_latency_seconds_count{measuring_point=\"mixer\",frame_age=\"0\",card=\"$card\",instance=~\"$instance\"}[1m])\n) without (frame_type)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Mixer latency, card $card", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_EXAMPLE}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 58 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "nageru_memory_used_bytes{instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Used", + "refId": "A", + "step": 30 + }, + { + "expr": "nageru_memory_locked_limit_bytes{instance=~\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Max locked", + "refId": "B", + "step": 30 + }, + { + "expr": "nageru_memory_gpu_used_bytes{instance=~\"$instance\"} ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "GPU used", + "refId": "C" + }, + { + "expr": "nageru_memory_gpu_total_bytes{instance=~\"$instance\"} ", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "GPU max", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "${DS_EXAMPLE}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "nageru_latency_seconds{measuring_point=\"mixer\"}", + "refresh": 1, + "regex": "/.*instance=\"([^\"]+)\".*/", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_EXAMPLE}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "card", + "options": [], + "query": "nageru_latency_seconds{measuring_point=\"mixer\"}", + "refresh": 1, + "regex": "/.*card=\"(\\d+)\".*/", + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Nageru", + "uid": "UML0ZDMmz", + "version": 7 +} diff --git a/README b/README new file mode 100644 index 0000000..e02ea23 --- /dev/null +++ b/README @@ -0,0 +1,266 @@ +Nageru is a live video mixer, based around the standard M/E workflow. + + +Features: + + - High performance on modest hardware (720p60 with two input streams + on my Thinkpad X240[1]); almost all pixel processing is done on the GPU. + + - High output quality; Lanczos3 scaling, subpixel precision everywhere, + white balance adjustment, mix of 16- and 32-bit floating point + for intermediate calculations, dithered output, optional 10-bit input + and output support. + + - Proper sound support: Syncing of multiple unrelated sources through + high-quality resampling, multichannel mixing with separate effects + per-bus, cue out for headphones, dynamic range compression, + three-band graphical EQ (pluss a fixed low-cut), level meters conforming + to EBU R128, automation via MIDI controllers. + + - Theme engine encapsulating the design demands of each individual + event; Lua code is responsible for setting up the pixel processing + pipelines, running transitions etc., so that the visual look is + consistent between operators. + + - HTML rendering (through Chromium Embedded Framework), for high-quality + and flexible overlay or other graphics. + + - Comprehensive monitoring through Prometheus metrics. + +[1] For reference, that is: Core i7 4600U (dualcore 2.10GHz, clocks down +to 800 MHz after 30 seconds due to thermal constraints), Intel HD Graphics +4400 (ie., without the extra L4 cache from Iris Pro), single-channel DDR3 RAM +(so 12.8 GB/sec theoretical memory bandwidth, shared between CPU and GPU). + + +Nageru currently needs: + + - An Intel processor with Intel Quick Sync, or otherwise some hardware + H.264 encoder exposed through VA-API. Note that you can use VA-API over + DRM instead of X11, to use a non-Intel GPU for rendering but still use + Quick Sync (Nageru does this automatically for you if needed). + + - Two or more Blackmagic USB3 or PCI cards, either HDMI or SDI. + The PCI cards need Blackmagic's own drivers installed. The USB3 cards + are driven through the “bmusb” driver, using libusb-1.0. If you want + zerocopy USB, you need libusb 1.0.21 or newer, as well as a recent + kernel (4.6.0 or newer). Zerocopy USB helps not only for performance, + but also for stability. You need at least version 0.7.0. + + - Movit, my GPU-based video filter library (https://movit.sesse.net). + You will need at least version 1.5.2. + + - Qt 5.5 or newer for the GUI. + + - QCustomPlot for the histogram display in the frame analyzer. + + - libmicrohttpd for the embedded web server. + + - x264 for encoding high-quality video suitable for streaming to end users. + + - ffmpeg for muxing, and for encoding audio. You will need at least + version 3.1. + + - Working OpenGL; Movit works with almost any modern OpenGL implementation. + Nageru has been tested with Intel on Mesa (you want 11.2 or newer, due + to critical stability bugfixes), and with NVIDIA's proprietary drivers. + The status of AMD's proprietary drivers is currently unknown. + + - libzita-resampler, for resampling sound sources so that they are in sync + between sources, and also for oversampling for the peak meter. + + - LuaJIT, for driving the theme engine. + + - Meson, for building. + + - Optional: CEF (Chromium Embedded Framework), for HTML graphics. + If you build without CEF, the HTMLInput class will not be available from + the theme. You can get binary downloads of CEF from + + http://opensource.spotify.com/cefbuilds/index.html + + Simply download the right build for your platform (the “minimal” build + is fine) and add -Dcef_dir=/cef_binary_X.XXXX.XXXX.XXXXXXXX_linux64 + on the meson command line (substituting X with the real version as required). + + +If on Debian stretch or something similar, you can install everything you need +with: + + apt install qtbase5-dev libqt5opengl5-dev qt5-default libqcustomplot-dev \ + pkg-config libmicrohttpd-dev libusb-1.0-0-dev libluajit-5.1-dev \ + libzita-resampler-dev libva-dev libavcodec-dev libavformat-dev \ + libswscale-dev libavresample-dev libmovit-dev libegl1-mesa-dev \ + libasound2-dev libx264-dev libbmusb-dev protobuf-compiler \ + libprotobuf-dev + +Exceptions as of November 2018: + + - You will need Movit from testing or unstable; stretch only has 1.4.0. + + - You will need bmusb from testing or unstable; stretch only has 0.5.4. + + - You will need a Meson backport; the version in stretch is too old. + + - Debian does not carry CEF (but it is optional). You can get experimental + (and not security-supported) CEF Debian packages built for unstable at + http://storage.sesse.net/cef/, and then configure Nageru with + + meson obj -Dcef_dir=/usr/lib/x86_64-linux-gnu/cef -Dcef_build_type=system -Dcef_no_icudtl=true + +The patches/ directory contains a patch that helps zita-resampler performance. +It is meant for upstream, but was not in at the time Nageru was released. +It is taken to be by Steinar H. Gunderson (ie., my ex-work +email), and under the same license as zita-resampler itself. + +Nageru uses Meson to build. For a default build, type + + meson obj && cd obj && ninja + +To start it, just hook up your equipment, and then type “./nageru”. + +It is strongly recommended to have the rights to run at real-time priority; +it will make the USB3 threads do so, which will make them a lot more stable. +(A reasonable hack for testing is probably just to run it as root using sudo, +although you might not want to do that in production.) Note also that if you +are running a desktop compositor, it will steal significant amounts of GPU +performance. The same goes for PulseAudio. + +Nageru will open a HTTP server at port 9095, where you can extract a live +H264+PCM signal in nut mux (e.g. http://127.0.0.1:9095/stream.nut). +It is probably too high bitrate (~25 Mbit/sec depending on content) to send to +users, but you can easily send it around in your internal network and then +transcode it in e.g. VLC. A copy of the stream (separately muxed) will also +be saved live to local disk. + +If you have a fast CPU (typically a quadcore desktop; most laptops will spend +most of their CPU on running Nageru itself), you can use x264 for the outgoing +stream instead of Quick Sync; it is much better quality for the same bitrate, +and also has proper bitrate controls. Simply add --http-x264-video on the +command line. (You may also need to add something like "--x264-preset veryfast", +since the default "medium" preset might be too CPU-intensive, but YMMV.) +The stream saved to disk will still be the Quick Sync-encoded stream, as it is +typically higher bitrate and thus also higher quality. Note that if you add +".metacube" at the end of the URL (e.g. "http://127.0.0.1:9095/stream.ts.metacube"), +you will get a stream suitable for streaming through the Cubemap video reflector +(cubemap.sesse.net). A typical example would be: + + ./nageru --http-x264-video --x264-preset veryfast --x264-tune film \ + --http-mux mp4 --http-audio-codec libfdk_aac --http-audio-bitrate 128 + +If you are comfortable with using all your remaining CPU power on the machine +for x264, try --x264-speedcontrol, which will try to adjust the preset +dynamically for maximum quality, at the expense of somewhat higher delay. + +See --help for more information on options in general. + +The name “Nageru” is a play on the Japanese verb 投げる (nageru), which means +to throw or cast. (I also later learned that it could mean to face defeat or +give up, but that's not the intended meaning.) + + +Nageru's home page is at https://nageru.sesse.net/, where you can also find +contact information, full documentation and link to the latest version. + + +Legalese: TL;DR: Everything is GPLv3-or-newer compatible, and see +Intel's copyright license at quicksync_encoder.h. + + +Nageru is Copyright (C) 2015 Steinar H. Gunderson . +Portions Copyright (C) 2003 Rune Holm. +Portions Copyright (C) 2010-2015 Fons Adriaensen . +Portions Copyright (C) 2012-2015 Fons Adriaensen . +Portions Copyright (C) 2008-2015 Fons Adriaensen . +Portions Copyright (c) 2007-2013 Intel Corporation. All Rights Reserved. + + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + + +Portions of quicksync_encoder.h and quicksync_encoder.cpp: + +Copyright (c) 2007-2013 Intel Corporation. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sub license, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice (including the +next paragraph) shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. +IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR +ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +All files in decklink/: + +Copyright (c) 2009 Blackmagic Design +Copyright (c) 2015 Blackmagic Design + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +Marked parts of theme.cpp (Lua shims): + +The MIT License (MIT) + +Copyright (c) 2013 Hisham Muhammad + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/bmusb b/bmusb new file mode 160000 index 0000000..5163d25 --- /dev/null +++ b/bmusb @@ -0,0 +1 @@ +Subproject commit 5163d25c65c3028090db1aea6587ec2fb4cb823e diff --git a/experiments/measure-x264.pl b/experiments/measure-x264.pl new file mode 100644 index 0000000..1ba2d9e --- /dev/null +++ b/experiments/measure-x264.pl @@ -0,0 +1,121 @@ +#! /usr/bin/perl + +# +# A script to measure the quality and speed of the x264 presets used in speed control. +# + +use strict; +use warnings; +use Time::HiRes; + +my $ssim_mode = 1; +my $output_cpp = 1; +my $flags = "--bitrate 4000 --frames 1000"; +my $override_flags = "--weightp 1 --mbtree --rc-lookahead 20 --b-adapt 1 --bframes 3"; +my $file = "elephants_dream_1080p24.y4m"; # https://media.xiph.org/video/derf/y4m/elephants_dream_1080p24.y4m + +if ($ssim_mode) { + # This can be run on a faster machine if you want to. It just measures SSIM; + # don't trust the timings, not even which modes are faster than others. + # The mode where $output_cpp=0 is just meant as a quick way to test new presets + # to see if they are good candidates. + $flags .= " --threads 40 --ssim"; + $override_flags .= " --tune ssim"; + open my $fh, "<", "presets.txt" + or die "presets.txt: $!"; + my $preset_num = 0; + for my $preset (<$fh>) { + chomp $preset; + my ($ssim, $elapsed) = measure_preset($file, $flags, $override_flags, $preset); + if ($output_cpp) { + output_cpp($file, $flags, $override_flags, $preset, $ssim, $preset_num++); + } else { + printf "%sdb %.3f %s\n", $ssim, $elapsed, $preset; + } + } + close $fh; +} else { + # Actual benchmarking. + my $repeat = 1; + $flags .= " --threads 4"; + open my $fh, "<", "presets.txt" + or die "presets.txt: $!"; + my $base = undef; + for my $preset (<$fh>) { + chomp $preset; + my $sum_elapsed = 0.0; + for my $i (1..$repeat) { + my (undef, $elapsed) = measure_preset($file, $flags, $override_flags, $preset); + $sum_elapsed += $elapsed; + } + my $avg = $sum_elapsed / $repeat; + $base //= $avg; + printf "%.3f %s\n", $avg / $base, $preset; + } + close $fh; +} + +sub measure_preset { + my ($file, $flags, $override_flags, $preset) = @_; + + my $now = [Time::HiRes::gettimeofday]; + my $ssim; + open my $x264, "-|", "/usr/bin/x264 $flags $preset $override_flags -o /dev/null $file 2>&1"; + for my $line (<$x264>) { + $line =~ /SSIM Mean.*\(\s*(\d+\.\d+)db\)/ and $ssim = $1; + } + close $x264; + my $elapsed = Time::HiRes::tv_interval($now); + return ($ssim, $elapsed); +} + +sub output_cpp { + my ($file, $flags, $override_flags, $preset, $ssim, $preset_num) = @_; + unlink("tmp.h264"); + system("/usr/bin/x264 $flags $preset $override_flags --frames 1 -o tmp.h264 $file >/dev/null 2>&1"); + open my $fh, "<", "tmp.h264" + or die "tmp.h264: $!"; + my $raw; + { + local $/ = undef; + $raw = <$fh>; + } + close $fh; + + $raw =~ /subme=(\d+)/ or die; + my $subme = $1; + + $raw =~ /me=(\S+)/ or die; + my $me = "X264_ME_" . uc($1); + + $raw =~ /ref=(\d+)/ or die; + my $refs = $1; + + $raw =~ /mixed_ref=(\d+)/ or die; + my $mix = $1; + + $raw =~ /trellis=(\d+)/ or die; + my $trellis = $1; + + $raw =~ /analyse=0x[0-9a-f]+:(0x[0-9a-f]+)/ or die; + my $partitions_hex = oct($1); + my @partitions = (); + push @partitions, 'I8' if ($partitions_hex & 0x0002); + push @partitions, 'I4' if ($partitions_hex & 0x0001); + push @partitions, 'P8' if ($partitions_hex & 0x0010); + push @partitions, 'B8' if ($partitions_hex & 0x0100); + push @partitions, 'P4' if ($partitions_hex & 0x0020); + my $partitions = join('|', @partitions); + + $raw =~ /direct=(\d+)/ or die; + my $direct = $1; + + $raw =~ /me_range=(\d+)/ or die; + my $merange = $1; + + print "\n"; + print "\t// Preset $preset_num: ${ssim}db, $preset\n"; + print "\t{ .time= 0.000, .subme=$subme, .me=$me, .refs=$refs, .mix=$mix, .trellis=$trellis, .partitions=$partitions, .direct=$direct, .merange=$merange },\n"; + +#x264 - core 148 r2705 3f5ed56 - H.264/MPEG-4 AVC codec - Copyleft 2003-2016 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=34 lookahead_threads=5 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=250 keyint_min=24 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00 +} diff --git a/experiments/presets.txt b/experiments/presets.txt new file mode 100644 index 0000000..1ef2997 --- /dev/null +++ b/experiments/presets.txt @@ -0,0 +1,23 @@ +--preset superfast +--preset superfast --subme 2 +--preset veryfast +--preset veryfast --subme 3 +--preset veryfast --subme 3 --ref 2 +--preset veryfast --subme 4 --ref 2 +--preset faster +--preset faster --mixed-refs +--preset faster --mixed-refs --subme 5 +--preset fast +--preset fast --subme 7 +--preset medium +--preset medium --subme 8 +--preset medium --subme 8 --trellis 2 +--preset medium --subme 8 --trellis 2 --direct auto +--preset slow +--preset slow --subme 9 +--preset slow --subme 9 --me umh +--preset slow --subme 9 --me umh --ref 6 +--preset slow --subme 9 --me umh --ref 7 +--preset slower +--preset slower --subme 10 +--preset veryslow diff --git a/experiments/queue_drop_policy.cpp b/experiments/queue_drop_policy.cpp new file mode 100644 index 0000000..dfd5f3c --- /dev/null +++ b/experiments/queue_drop_policy.cpp @@ -0,0 +1,588 @@ +/* + * A program to simulate various queue-drop strategies, using real frame + * arrival data as input. Contains various anchors, as well as parametrized + * values of the real algorithms that have been used in Nageru over time. + * + * Expects a log of frame arrivals (in and out). This isn't included in the + * git repository because it's quite large, but there's one available + * in compressed form at + * + * https://storage.sesse.net/nageru-latency-log.txt.xz + * + * The data set in question contains a rather difficult case, with two 50 Hz + * clocks slowly drifting from each other (at the rate of about a frame an hour). + * This means they are very nearly in sync for a long time, where rare bursts + * of jitter can make it hard for the algorithm to find the right level of + * conservatism. + * + * This is not meant to be production-quality code. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +size_t max_drops = numeric_limits::max(); +size_t max_underruns = numeric_limits::max(); +double max_latency_ms = numeric_limits::max(); + +struct Event { + enum { IN, OUT } direction; + double t; +}; + +class Queue { +public: + void add_frame(double t); + void get_frame(double now); + void drop_frame(); + void eval(const string &name); + size_t queue_len() const { return frames_in_queue.size(); } + bool should_abort() const { return num_underruns > max_underruns || num_drops > max_drops; } + +private: + deque frames_in_queue; + size_t num_underruns = 0; + size_t num_drops = 0; + size_t frames_since_underrun = 0; + size_t num_drops_on_first = 0; + + double latency_sum = 0.0; + size_t latency_count = 0; +}; + +void Queue::add_frame(double t) +{ + frames_in_queue.push_back(t); +} + +void Queue::get_frame(double now) +{ + if (frames_in_queue.empty()) { + ++num_underruns; + frames_since_underrun = 0; + return; + } + double t = frames_in_queue.front(); + frames_in_queue.pop_front(); + assert(now >= t); + latency_sum += (now - t); + ++latency_count; + ++frames_since_underrun; +} + +void Queue::drop_frame() +{ + assert(!frames_in_queue.empty()); + frames_in_queue.pop_front(); + ++num_drops; + if (frames_since_underrun <= 1) { + ++num_drops_on_first; + } +} + +void Queue::eval(const string &name) +{ + double latency_ms = 1e3 * latency_sum / latency_count; + if (num_underruns > max_underruns) return; + if (num_drops > max_drops) return; + if (latency_ms > max_latency_ms) return; + printf("%-50s: %2lu frames left in queue at end, %5lu underruns, %5lu drops (%5lu immediate), %6.2f ms avg latency\n", + name.c_str(), frames_in_queue.size(), num_underruns, num_drops, num_drops_on_first, latency_ms); +} + +// A strategy that never drops; low anchor for drops and underruns, high anchor for latency. +void test_nodrop(const vector &events) +{ + Queue q; + for (const Event &event : events) { + if (event.direction == Event::IN) { + q.add_frame(event.t); + } else { + q.get_frame(event.t); + } + } + q.eval("no-drop"); +} + +// A strategy that accepts only one element in the queue; low anchor for latency. +void test_limit_to_1(const vector &events) +{ + Queue q; + for (const Event &event : events) { + if (event.direction == Event::IN) { + q.add_frame(event.t); + while (q.queue_len() > 1) q.drop_frame(); + } else { + q.get_frame(event.t); + } + } + q.eval("limit-to-1"); +} + +// A strategy that accepts one or two elements in the queue. +void test_limit_to_2(const vector &events) +{ + Queue q; + for (const Event &event : events) { + if (event.direction == Event::IN) { + q.add_frame(event.t); + while (q.queue_len() > 2) q.drop_frame(); + } else { + q.get_frame(event.t); + } + } + q.eval("limit-to-2"); +} + +// The algorithm used from Nageru 1.2.0 to 1.6.0; raise the ceiling by 1 every time +// we underrun, drop it if the ceiling hasn't been needed for 1000 frames. +void test_nageru_1_2_0(const vector &events) +{ + Queue q; + unsigned safe_queue_length = 1; + unsigned frames_with_at_least_one = 0; + bool been_at_safe_point_since_last_starvation = false; + for (const Event &event : events) { + if (event.direction == Event::IN) { + q.add_frame(event.t); + } else { + unsigned queue_length = q.queue_len(); + if (queue_length == 0) { // Starvation. + if (been_at_safe_point_since_last_starvation /*&& safe_queue_length < unsigned(global_flags.max_input_queue_frames)*/) { + ++safe_queue_length; + } + frames_with_at_least_one = 0; + been_at_safe_point_since_last_starvation = false; + q.get_frame(event.t); // mark it + continue; + } + if (queue_length >= safe_queue_length) { + been_at_safe_point_since_last_starvation = true; + } + if (++frames_with_at_least_one >= 1000 && safe_queue_length > 1) { + --safe_queue_length; + frames_with_at_least_one = 0; + } + while (q.queue_len() > safe_queue_length) { + q.drop_frame(); + } + q.get_frame(event.t); + } + } + q.eval("nageru-1.2.0"); +} + +class Jitter { + const double multiplier, alpha; + double expected_timestamp = -1.0; + double max_jitter_seconds = 0.0; + +public: + Jitter(double multiplier, double alpha) + : multiplier(multiplier), alpha(alpha) {} + + void update(double timestamp, double frame_duration, size_t dropped_frames) + { + if (expected_timestamp >= 0.0) { + expected_timestamp += dropped_frames * frame_duration; + double jitter_seconds = fabs(expected_timestamp - timestamp); + max_jitter_seconds = max(multiplier * jitter_seconds, alpha * max_jitter_seconds); // About two seconds half-time. + + // Cap at 100 ms. + max_jitter_seconds = min(max_jitter_seconds, 0.1); + } + expected_timestamp = timestamp + frame_duration; + } + + double get_expected() const + { + return expected_timestamp; + } + + double get_jitter() const + { + return max_jitter_seconds; + } +}; + +// Keep a running estimate of k times max jitter seen, decreasing by a factor alpha every frame. +void test_jitter_filter(const vector &events, double multiplier, double alpha, double margin) +{ + Queue q; + Jitter input_jitter(multiplier, alpha); + Jitter output_jitter(multiplier, alpha); + + for (const Event &event : events) { + if (event.direction == Event::IN) { + input_jitter.update(event.t, 0.020, 0); + q.add_frame(event.t); + } else { + double now = event.t; + output_jitter.update(event.t, 0.020, 0); + q.get_frame(event.t); + + double seconds_until_next_frame = max(input_jitter.get_expected() - now + input_jitter.get_jitter(), 0.0); + double master_frame_length_seconds = 0.020; + + seconds_until_next_frame += margin; // Hack. + + size_t safe_queue_length = max(floor((seconds_until_next_frame + output_jitter.get_jitter()) / master_frame_length_seconds), 0); + while (q.queue_len() > safe_queue_length) { + q.drop_frame(); + } + } + if (q.should_abort()) return; + } + + char name[256]; + snprintf(name, sizeof(name), "jitter-filter[mul=%.1f,alpha=%.4f,margin=%.1f]", multiplier, alpha, 1e3 * margin); + q.eval(name); +} + +// Implements an unbalanced binary search tree that can also satisfy order queries +// (e.g. “give me the 86th largest entry”). +class HistoryJitter { + const size_t history_length; + const double multiplier, percentile; + double expected_timestamp = 0.0; + double max_jitter_seconds = 0.0; + size_t num_updates = 0; + + deque history; + struct TreeNode { + double val; + size_t children = 0; + unique_ptr left, right; + }; + unique_ptr root; + + unique_ptr alloc_cache; // Holds the last freed value, for fast reallocation. + + TreeNode *alloc_node() + { + if (alloc_cache == nullptr) { + return new TreeNode; + } + alloc_cache->children = 0; + return alloc_cache.release(); + } + + void insert(double val) + { + if (root == nullptr) { + root.reset(alloc_node()); + root->val = val; + return; + } else { + insert(root.get(), val); + } + } + + void insert(TreeNode *node, double val) + { + ++node->children; + if (val <= node->val) { + // Goes into left. + if (node->left == nullptr) { + node->left.reset(alloc_node()); + node->left->val = val; + } else { + insert(node->left.get(), val); + } + } else { + // Goes into right. + if (node->right == nullptr) { + node->right.reset(alloc_node()); + node->right->val = val; + } else { + insert(node->right.get(), val); + } + } + } + + void remove(double val) + { + assert(root != nullptr); + if (root->children == 0) { + assert(root->val == val); + alloc_cache = move(root); + } else { + remove(root.get(), val); + } + } + + void remove(TreeNode *node, double val) + { + //printf("Down into %p looking for %f [left=%p right=%p]\n", node, val, node->left.get(), node->right.get()); + if (node->val == val) { + remove(node); + } else if (val < node->val) { + assert(node->left != nullptr); + --node->children; + if (node->left->children == 0) { + assert(node->left->val == val); + alloc_cache = move(node->left); + } else { + remove(node->left.get(), val); + } + } else { + assert(node->right != nullptr); + --node->children; + if (node->right->children == 0) { + assert(node->right->val == val); + alloc_cache = move(node->right); + } else { + remove(node->right.get(), val); + } + } + } + + // Declares a node to be empty, so it should pull up the value of one of its children. + // The node must be an interior node (ie., have at least one child). + void remove(TreeNode *node) + { + //printf("Decided that %p must be removed\n", node); + assert(node->children > 0); + --node->children; + + bool remove_left; + if (node->right == nullptr) { + remove_left = true; + } else if (node->left == nullptr) { + remove_left = false; + } else { + remove_left = (node->left->children >= node->right->children); + } + if (remove_left) { + if (node->left->children == 0) { + node->val = node->left->val; + alloc_cache = move(node->left); + } else { + // Move maximum value up to this node. + node->val = elem_at(node->left.get(), node->left->children); + remove(node->left.get(), node->val); + } + } else { + if (node->right->children == 0) { + node->val = node->right->val; + alloc_cache = move(node->right); + } else { + // Move minimum value up to this node. + node->val = elem_at(node->right.get(), 0); + remove(node->right.get(), node->val); + } + } + } + + double elem_at(size_t elem_idx) + { + return elem_at(root.get(), elem_idx); + } + + double elem_at(TreeNode *node, size_t elem_idx) + { + //printf("Looking for %lu in node %p [%lu children]\n", elem_idx, node, node->children); + assert(node != nullptr); + assert(elem_idx <= node->children); + if (node->left != nullptr) { + if (elem_idx <= node->left->children) { + return elem_at(node->left.get(), elem_idx); + } else { + elem_idx -= node->left->children + 1; + } + } + if (elem_idx == 0) { + return node->val; + } + return elem_at(node->right.get(), elem_idx - 1); + } + + void print_tree(TreeNode *node, size_t indent, double min, double max) + { + if (node == nullptr) return; + if (!(node->val >= min && node->val <= max)) { + //printf("node %p is outside range [%f,%f]\n", node, min, max); + assert(false); + } + for (size_t i = 0; i < indent * 2; ++i) putchar(' '); + printf("%f [%p, %lu children]\n", node->val, node, node->children); + print_tree(node->left.get(), indent + 1, min, node->val); + print_tree(node->right.get(), indent + 1, node->val, max); + } + +public: + HistoryJitter(size_t history_length, double multiplier, double percentile) + : history_length(history_length), multiplier(multiplier), percentile(percentile) {} + + void update(double timestamp, double frame_duration, size_t dropped_frames) + { + //if (++num_updates % 1000 == 0) { + // printf("%d... [%lu in tree %p]\n", num_updates, root->children + 1, root.get()); + //} + + if (expected_timestamp >= 0.0) { + expected_timestamp += dropped_frames * frame_duration; + double jitter_seconds = fabs(expected_timestamp - timestamp); + + history.push_back(jitter_seconds); + insert(jitter_seconds); + //printf("\nTree %p after insert of %f:\n", root.get(), jitter_seconds); + //print_tree(root.get(), 0, -HUGE_VAL, HUGE_VAL); + while (history.size() > history_length) { + // printf("removing %f, because %p has %lu elements and history has %lu elements\n", history.front(), root.get(), root->children + 1, history.size()); + remove(history.front()); + history.pop_front(); + } + + size_t elem_idx = lrint(percentile * (history.size() - 1)); +// printf("Searching for element %lu in %p, which has %lu elements (history has %lu elements)\n", elem_idx, root.get(), root->children + 1, history.size()); +// fflush(stdout); +// + // Cap at 100 ms. + max_jitter_seconds = min(elem_at(elem_idx), 0.1); + } + expected_timestamp = timestamp + frame_duration; + } + + double get_expected() const + { + return expected_timestamp; + } + + double get_jitter() const + { + return max_jitter_seconds * multiplier; + } +}; + +void test_jitter_history(const vector &events, size_t history_length, double multiplier, double percentile, double margin) +{ + Queue q; + HistoryJitter input_jitter(history_length, multiplier, percentile); + HistoryJitter output_jitter(history_length, multiplier, percentile); + + for (const Event &event : events) { + if (event.direction == Event::IN) { + input_jitter.update(event.t, 0.020, 0); + q.add_frame(event.t); + } else { + double now = event.t; + output_jitter.update(event.t, 0.020, 0); + q.get_frame(event.t); + + double seconds_until_next_frame = max(input_jitter.get_expected() - now + input_jitter.get_jitter(), 0.0); + double master_frame_length_seconds = 0.020; + + seconds_until_next_frame += margin; // Hack. + + size_t safe_queue_length = max(floor((seconds_until_next_frame + output_jitter.get_jitter()) / master_frame_length_seconds), 0); + while (q.queue_len() > safe_queue_length) { + q.drop_frame(); + } + } + if (q.should_abort()) return; + } + char name[256]; + snprintf(name, sizeof(name), "history[len=%lu,mul=%.1f,pct=%.4f,margin=%.1f]", history_length, multiplier, percentile, 1e3 * margin); + q.eval(name); +} + +int main(int argc, char **argv) +{ + static const option long_options[] = { + { "max-drops", required_argument, 0, 'd' }, + { "max-underruns", required_argument, 0, 'u' }, + { "max-latency-ms", required_argument, 0, 'l' }, + { 0, 0, 0, 0 } + }; + for ( ;; ) { + int option_index = 0; + int c = getopt_long(argc, argv, "d:u:l:", long_options, &option_index); + + if (c == -1) { + break; + } + switch (c) { + case 'd': + max_drops = atof(optarg); + break; + case 'u': + max_underruns = atof(optarg); + break; + case 'l': + max_latency_ms = atof(optarg); + break; + default: + fprintf(stderr, "Usage: simul [--max-drops NUM] [--max-underruns NUM] [--max-latency-ms TIME]\n"); + exit(1); + } + } + + vector events; + + const char *filename = (optind < argc) ? argv[optind] : "nageru-latency-log.txt"; + FILE *fp = fopen(filename, "r"); + if (fp == nullptr) { + perror(filename); + exit(1); + } + while (!feof(fp)) { + char dir[256]; + double t; + + if (fscanf(fp, "%s %lf", dir, &t) != 2) { + break; + } + if (dir[0] == 'I') { + events.push_back(Event{Event::IN, t}); + } else if (dir[0] == 'O') { + events.push_back(Event{Event::OUT, t}); + } else { + fprintf(stderr, "ERROR: Unreadable line\n"); + exit(1); + } + } + fclose(fp); + + sort(events.begin(), events.end(), [](const Event &a, const Event &b) { return a.t < b.t; }); + + test_nodrop(events); + test_limit_to_1(events); + test_limit_to_2(events); + test_nageru_1_2_0(events); + for (double multiplier : { 0.0, 0.5, 1.0, 2.0, 3.0, 5.0 }) { + for (double alpha : { 0.5, 0.9, 0.99, 0.995, 0.999, 0.9999 }) { + for (double margin_ms : { -1.0, 0.0, 1.0, 2.0, 5.0, 10.0, 20.0 }) { + test_jitter_filter(events, multiplier, alpha, 1e-3 * margin_ms); + } + } + } + for (size_t history_samples : { 10, 100, 500, 1000, 5000, 10000, 25000 }) { + for (double multiplier : { 0.5, 1.0, 2.0, 3.0, 5.0, 10.0 }) { + for (double percentile : { 0.5, 0.75, 0.9, 0.99, 0.995, 0.999, 1.0 }) { + if (lrint(percentile * (history_samples - 1)) == int(history_samples - 1) && percentile != 1.0) { + // Redundant. + continue; + } + + //for (double margin_ms : { -1.0, 0.0, 1.0, 2.0, 5.0, 10.0, 20.0 }) { + for (double margin_ms : { 0.0 }) { + test_jitter_history(events, history_samples, multiplier, percentile, 1e-3 * margin_ms); + } + } + } + } +} diff --git a/futatabi/meson.build b/futatabi/meson.build index a9c95b4..fdcc446 100644 --- a/futatabi/meson.build +++ b/futatabi/meson.build @@ -18,13 +18,6 @@ vadrmdep = dependency('libva-drm') vax11dep = dependency('libva-x11') x11dep = dependency('x11') -# Add the right MOVIT_SHADER_DIR definition. -r = run_command('pkg-config', '--variable=shaderdir', 'movit') -if r.returncode() != 0 - error('Movit pkg-config installation is broken.') -endif -add_global_arguments('-DMOVIT_SHADER_DIR="' + r.stdout().strip() + '"', language: 'cpp') - # Protobuf compilation. gen = generator(protoc, \ output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], diff --git a/meson.build b/meson.build index 6c13d8c..8357005 100644 --- a/meson.build +++ b/meson.build @@ -1,2 +1,11 @@ -project('futatabi', 'cpp') +project('nageru', 'cpp', default_options: ['buildtype=debugoptimized']) + +# Add the right MOVIT_SHADER_DIR definition. +r = run_command('pkg-config', '--variable=shaderdir', 'movit') +if r.returncode() != 0 + error('Movit pkg-config installation is broken.') +endif +add_project_arguments('-DMOVIT_SHADER_DIR="' + r.stdout().strip() + '"', language: 'cpp') + +subdir('nageru') subdir('futatabi') diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 0000000..115ba46 --- /dev/null +++ b/meson_options.txt @@ -0,0 +1,7 @@ +option('embedded_bmusb', type: 'boolean', value: false, description: 'Use bmusb from the bmusb/ git submodule instead of from the system') + +# Set this to build with CEF. +# E.g.: meson configure -Dcef_dir=/home/sesse/cef_binary_3.3282.1734.g8f26fe0_linux64 +option('cef_dir', type: 'string', description: 'If not empty, build against CEF in this directory') +option('cef_build_type', type: 'string', value: 'Release', description: 'CEF version to build against (Release or Debug, or “system” for a system-installed)') +option('cef_no_icudtl', type: 'boolean', value: false, description: 'Set to true if the CEF installation has no icudtl.dat.') diff --git a/nageru/aboutdialog.cpp b/nageru/aboutdialog.cpp new file mode 100644 index 0000000..94ef345 --- /dev/null +++ b/nageru/aboutdialog.cpp @@ -0,0 +1,16 @@ +#include "aboutdialog.h" + +#include + +#include "ui_aboutdialog.h" + +using namespace std; + +AboutDialog::AboutDialog() + : ui(new Ui::AboutDialog) +{ + ui->setupUi(this); + + connect(ui->button_box, &QDialogButtonBox::accepted, [this]{ this->close(); }); +} + diff --git a/nageru/aboutdialog.h b/nageru/aboutdialog.h new file mode 100644 index 0000000..5100c00 --- /dev/null +++ b/nageru/aboutdialog.h @@ -0,0 +1,24 @@ +#ifndef _ABOUTDIALOG_H +#define _ABOUTDIALOG_H 1 + +#include +#include + +class QObject; + +namespace Ui { +class AboutDialog; +} // namespace Ui + +class AboutDialog : public QDialog +{ + Q_OBJECT + +public: + AboutDialog(); + +private: + Ui::AboutDialog *ui; +}; + +#endif // !defined(_ABOUTDIALOG_H) diff --git a/nageru/aboutdialog.ui b/nageru/aboutdialog.ui new file mode 100644 index 0000000..57caf43 --- /dev/null +++ b/nageru/aboutdialog.ui @@ -0,0 +1,181 @@ + + + AboutDialog + + + + 0 + 0 + 684 + 544 + + + + About Nageru + + + + + + <p><b>Nageru 1.7.5</b></p> + +<p>Realtime video mixer</p> + + + + + + + true + + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> +<html><head><meta name="qrichtext" content="1" /></head><body> +<p> +Nageru is Copyright (C) 2015 Steinar H. Gunderson &lt;steinar+nageru@gunderson.no&gt;<br /> +Portions Copyright (C) 2003 Rune Holm.<br /> +Portions Copyright (C) 2010-2011 Fons Adriaensen &lt;fons@linuxaudio.org&gt;<br /> +Portions Copyright (C) 2012-2015 Fons Adriaensen &lt;fons@linuxaudio.org&gt;<br /> +Portions Copyright (C) 2008-2015 Fons Adriaensen &lt;fons@linuxaudio.org&gt;<br /> +Portions Copyright (c) 2007-2013 Intel Corporation. All Rights Reserved.</p> + +<p>This program is free software: you can redistribute it and/or modify<br /> +it under the terms of the GNU General Public License as published by<br /> +the Free Software Foundation, either version 3 of the License, or<br /> +(at your option) any later version.</p> + +<p>This program is distributed in the hope that it will be useful,<br /> +but WITHOUT ANY WARRANTY; without even the implied warranty of<br /> +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the<br /> +GNU General Public License for more details.</p> + +<p>You should have received a copy of the GNU General Public License<br /> +along with this program. If not, see &lt;<a href="http://www.gnu.org/licenses/"><span style=" text-decoration: underline; color:#0000ff;">http://www.gnu.org/licenses/</span></a>&gt;.</p> + +<p><br />Portions of h264encode.h and h264encode.cpp:</p> + +<p>Copyright (c) 2007-2013 Intel Corporation. All Rights Reserved.</p> + +<p>Permission is hereby granted, free of charge, to any person obtaining a<br /> +copy of this software and associated documentation files (the<br /> +&quot;Software&quot;), to deal in the Software without restriction, including<br /> +without limitation the rights to use, copy, modify, merge, publish,<br /> +distribute, sub license, and/or sell copies of the Software, and to<br /> +permit persons to whom the Software is furnished to do so, subject to<br /> +the following conditions:</p> + +<p>The above copyright notice and this permission notice (including the<br /> +next paragraph) shall be included in all copies or substantial portions<br /> +of the Software.</p> + +<p>THE SOFTWARE IS PROVIDED &quot;AS IS&quot;, WITHOUT WARRANTY OF ANY KIND, EXPRESS<br /> +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF<br /> +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.<br /> +IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR<br /> +ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,<br /> +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE<br /> +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.</p> + +<p><br />All files in decklink/:</p> + +<p>Copyright (c) 2009 Blackmagic Design<br /> +Copyright (c) 2015 Blackmagic Design</p> + +<p>Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following:</p> + +<p>The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor.</p> + +<p>THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE.</p> + +<p>Marked parts of theme.cpp (Lua shims):</p> + +<p>The MIT License (MIT)</p> + +<p>Copyright (c) 2013 Hisham Muhammad</p> + +<p>Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions:</p> + +<p>The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software.</p> + +<p>THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.</p> + +</body></html> + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Ok + + + + + + + + + button_box + accepted() + Dialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + button_box + rejected() + Dialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + diff --git a/nageru/akai_midimix.midimapping b/nageru/akai_midimix.midimapping new file mode 100644 index 0000000..bb219cc --- /dev/null +++ b/nageru/akai_midimix.midimapping @@ -0,0 +1,402 @@ +# Example mapping for the Akai MIDImix. This one is written by hand, +# and serves as a simple example of the basic features. The MIDImix +# doesn't have a ton of controls, so not everything is mapped up, +# and some "wrong" mappings need to be done; in particular, we've set up +# two controller banks and switch between them with the BANK LEFT and +# BANK RIGHT buttons (which are normally meant to switch between channels +# 1–8 and 9–16, as I understand it). +# +# The mappings for the 270° pots on each bus are: +# +# Bank 1: Treble, mid, bass +# Bank 2: Gain, compressor threshold, (globals) +# +# The “(globals)” here are only for use on the two rightmost buses: +# The third pot on bus 7 controls the lo-cut cutoff, and the pot on +# bus 8 controls the limiter threshold. +# +# The mute button controls muting (obviously) for that bus, and the solo +# button (accessible by holding the global solo button and pressing the +# mute button for the bus) is abused for toggling auto gain staging. +# +# The REC ARM button for each bus is abused to be a “has peaked” meter; +# pressing it will reset the measurement. +# +# Finally, the faders work pretty much as you'd expect; each bus' fader +# is connected to the volume for that bus, and the master fader is +# connected to the global makeup gain. + +num_controller_banks: 2 +treble_bank: 0 +mid_bank: 0 +bass_bank: 0 +gain_bank: 1 +compressor_threshold_bank: 1 +locut_bank: 1 +limiter_threshold_bank: 1 + +# Bus 1. We also store the master controller here. +bus_mapping { + treble { + controller_number: 16 + } + mid { + controller_number: 17 + } + bass { + controller_number: 18 + } + gain { + controller_number: 16 + } + compressor_threshold { + controller_number: 17 + } + fader { + controller_number: 19 + } + toggle_mute { + note_number: 1 + } + toggle_auto_gain_staging { + note_number: 2 + } + clear_peak { + note_number: 3 + } + + # Master. + makeup_gain { + controller_number: 62 + } + select_bank_1 { + note_number: 25 # Bank left. + } + select_bank_2 { + note_number: 26 # Bank right. + } + + # Lights. + is_muted { + note_number: 1 + } + auto_gain_staging_is_on { + note_number: 2 + } + has_peaked { + note_number: 3 + } + + # Global lights. + bank_1_is_selected { + note_number: 25 + } + bank_2_is_selected { + note_number: 26 + } +} + +# Bus 2. +bus_mapping { + treble { + controller_number: 20 + } + mid { + controller_number: 21 + } + bass { + controller_number: 22 + } + gain { + controller_number: 20 + } + compressor_threshold { + controller_number: 21 + } + fader { + controller_number: 23 + } + toggle_mute { + note_number: 4 + } + toggle_auto_gain_staging { + note_number: 5 + } + clear_peak { + note_number: 6 + } + + # Lights. + is_muted { + note_number: 4 + } + auto_gain_staging_is_on { + note_number: 5 + } + has_peaked { + note_number: 6 + } +} + +# Bus 3. +bus_mapping { + treble { + controller_number: 24 + } + mid { + controller_number: 25 + } + bass { + controller_number: 26 + } + gain { + controller_number: 24 + } + compressor_threshold { + controller_number: 25 + } + fader { + controller_number: 27 + } + toggle_mute { + note_number: 7 + } + toggle_auto_gain_staging { + note_number: 8 + } + clear_peak { + note_number: 9 + } + + # Lights. + is_muted { + note_number: 7 + } + auto_gain_staging_is_on { + note_number: 8 + } + has_peaked { + note_number: 9 + } +} + +# Bus 4. +bus_mapping { + treble { + controller_number: 28 + } + mid { + controller_number: 29 + } + bass { + controller_number: 30 + } + gain { + controller_number: 28 + } + compressor_threshold { + controller_number: 29 + } + fader { + controller_number: 31 + } + toggle_mute { + note_number: 10 + } + toggle_auto_gain_staging { + note_number: 11 + } + clear_peak { + note_number: 12 + } + + # Lights. + is_muted { + note_number: 10 + } + auto_gain_staging_is_on { + note_number: 11 + } + has_peaked { + note_number: 12 + } +} + +# Bus 5. Note the discontinuity in the controller numbers, +# but not in the note numbers. +bus_mapping { + treble { + controller_number: 46 + } + mid { + controller_number: 47 + } + bass { + controller_number: 48 + } + gain { + controller_number: 46 + } + compressor_threshold { + controller_number: 47 + } + fader { + controller_number: 49 + } + toggle_mute { + note_number: 13 + } + toggle_auto_gain_staging { + note_number: 14 + } + clear_peak { + note_number: 15 + } + + # Lights. + is_muted { + note_number: 13 + } + auto_gain_staging_is_on { + note_number: 14 + } + has_peaked { + note_number: 15 + } +} + +# Bus 6. +bus_mapping { + treble { + controller_number: 50 + } + mid { + controller_number: 51 + } + bass { + controller_number: 52 + } + gain { + controller_number: 50 + } + compressor_threshold { + controller_number: 51 + } + fader { + controller_number: 53 + } + toggle_mute { + note_number: 16 + } + toggle_auto_gain_staging { + note_number: 17 + } + clear_peak { + note_number: 18 + } + + # Lights. + is_muted { + note_number: 16 + } + auto_gain_staging_is_on { + note_number: 17 + } + has_peaked { + note_number: 18 + } +} + +# Bus 7. +bus_mapping { + treble { + controller_number: 54 + } + mid { + controller_number: 55 + } + bass { + controller_number: 56 + } + gain { + controller_number: 54 + } + compressor_threshold { + controller_number: 55 + } + fader { + controller_number: 57 + } + toggle_mute { + note_number: 19 + } + toggle_auto_gain_staging { + note_number: 20 + } + clear_peak { + note_number: 21 + } + + # Lights. + is_muted { + note_number: 19 + } + auto_gain_staging_is_on { + note_number: 20 + } + has_peaked { + note_number: 21 + } + + # Global controllers. + locut { + controller_number: 56 + } +} + +# Bus 8. +bus_mapping { + treble { + controller_number: 58 + } + mid { + controller_number: 59 + } + bass { + controller_number: 60 + } + gain { + controller_number: 58 + } + compressor_threshold { + controller_number: 59 + } + fader { + controller_number: 61 + } + toggle_mute { + note_number: 22 + } + toggle_auto_gain_staging { + note_number: 23 + } + clear_peak { + note_number: 24 + } + + # Lights. + is_muted { + note_number: 22 + } + auto_gain_staging_is_on { + note_number: 23 + } + has_peaked { + note_number: 24 + } + + # Global controllers. + limiter_threshold { + controller_number: 60 + } +} diff --git a/nageru/alsa_input.cpp b/nageru/alsa_input.cpp new file mode 100644 index 0000000..08a67f7 --- /dev/null +++ b/nageru/alsa_input.cpp @@ -0,0 +1,266 @@ +#include "alsa_input.h" + +#include +#include +#include +#include +#include +#include + +#include "alsa_pool.h" +#include "bmusb/bmusb.h" +#include "timebase.h" + +using namespace std; +using namespace std::chrono; +using namespace std::placeholders; + +#define RETURN_ON_ERROR(msg, expr) do { \ + int err = (expr); \ + if (err < 0) { \ + fprintf(stderr, "[%s] " msg ": %s\n", device.c_str(), snd_strerror(err)); \ + if (err == -ENODEV) return CaptureEndReason::DEVICE_GONE; \ + return CaptureEndReason::OTHER_ERROR; \ + } \ +} while (false) + +#define RETURN_FALSE_ON_ERROR(msg, expr) do { \ + int err = (expr); \ + if (err < 0) { \ + fprintf(stderr, "[%s] " msg ": %s\n", device.c_str(), snd_strerror(err)); \ + return false; \ + } \ +} while (false) + +#define WARN_ON_ERROR(msg, expr) do { \ + int err = (expr); \ + if (err < 0) { \ + fprintf(stderr, "[%s] " msg ": %s\n", device.c_str(), snd_strerror(err)); \ + } \ +} while (false) + +ALSAInput::ALSAInput(const char *device, unsigned sample_rate, unsigned num_channels, audio_callback_t audio_callback, ALSAPool *parent_pool, unsigned internal_dev_index) + : device(device), + sample_rate(sample_rate), + num_channels(num_channels), + audio_callback(audio_callback), + parent_pool(parent_pool), + internal_dev_index(internal_dev_index) +{ +} + +bool ALSAInput::open_device() +{ + RETURN_FALSE_ON_ERROR("snd_pcm_open()", snd_pcm_open(&pcm_handle, device.c_str(), SND_PCM_STREAM_CAPTURE, 0)); + + // Set format. + snd_pcm_hw_params_t *hw_params; + snd_pcm_hw_params_alloca(&hw_params); + if (!set_base_params(device.c_str(), pcm_handle, hw_params, &sample_rate)) { + return false; + } + + RETURN_FALSE_ON_ERROR("snd_pcm_hw_params_set_channels()", snd_pcm_hw_params_set_channels(pcm_handle, hw_params, num_channels)); + + // Fragment size of 64 samples (about 1 ms at 48 kHz; a frame at 60 + // fps/48 kHz is 800 samples.) We ask for 64 such periods in our buffer + // (~85 ms buffer); more than that, and our jitter is probably so high + // that the resampling queue can't keep up anyway. + // The entire thing with periods and such is a bit mysterious to me; + // seemingly I can get 96 frames at a time with no problems even if + // the period size is 64 frames. And if I set num_periods to e.g. 1, + // I can't have a big buffer. + num_periods = 16; + int dir = 0; + RETURN_FALSE_ON_ERROR("snd_pcm_hw_params_set_periods_near()", snd_pcm_hw_params_set_periods_near(pcm_handle, hw_params, &num_periods, &dir)); + period_size = 64; + dir = 0; + RETURN_FALSE_ON_ERROR("snd_pcm_hw_params_set_period_size_near()", snd_pcm_hw_params_set_period_size_near(pcm_handle, hw_params, &period_size, &dir)); + buffer_frames = 64 * 64; + RETURN_FALSE_ON_ERROR("snd_pcm_hw_params_set_buffer_size_near()", snd_pcm_hw_params_set_buffer_size_near(pcm_handle, hw_params, &buffer_frames)); + RETURN_FALSE_ON_ERROR("snd_pcm_hw_params()", snd_pcm_hw_params(pcm_handle, hw_params)); + //snd_pcm_hw_params_free(hw_params); + + // Figure out which format the card actually chose. + RETURN_FALSE_ON_ERROR("snd_pcm_hw_params_current()", snd_pcm_hw_params_current(pcm_handle, hw_params)); + snd_pcm_format_t chosen_format; + RETURN_FALSE_ON_ERROR("snd_pcm_hw_params_get_format()", snd_pcm_hw_params_get_format(hw_params, &chosen_format)); + + audio_format.num_channels = num_channels; + audio_format.bits_per_sample = 0; + switch (chosen_format) { + case SND_PCM_FORMAT_S16_LE: + audio_format.bits_per_sample = 16; + break; + case SND_PCM_FORMAT_S24_LE: + audio_format.bits_per_sample = 24; + break; + case SND_PCM_FORMAT_S32_LE: + audio_format.bits_per_sample = 32; + break; + default: + assert(false); + } + audio_format.sample_rate = sample_rate; + //printf("num_periods=%u period_size=%u buffer_frames=%u sample_rate=%u bits_per_sample=%d\n", + // num_periods, unsigned(period_size), unsigned(buffer_frames), sample_rate, audio_format.bits_per_sample); + + buffer.reset(new uint8_t[buffer_frames * num_channels * audio_format.bits_per_sample / 8]); + + snd_pcm_sw_params_t *sw_params; + snd_pcm_sw_params_alloca(&sw_params); + RETURN_FALSE_ON_ERROR("snd_pcm_sw_params_current()", snd_pcm_sw_params_current(pcm_handle, sw_params)); + RETURN_FALSE_ON_ERROR("snd_pcm_sw_params_set_start_threshold", snd_pcm_sw_params_set_start_threshold(pcm_handle, sw_params, num_periods * period_size / 2)); + RETURN_FALSE_ON_ERROR("snd_pcm_sw_params()", snd_pcm_sw_params(pcm_handle, sw_params)); + + RETURN_FALSE_ON_ERROR("snd_pcm_nonblock()", snd_pcm_nonblock(pcm_handle, 1)); + RETURN_FALSE_ON_ERROR("snd_pcm_prepare()", snd_pcm_prepare(pcm_handle)); + return true; +} + +bool ALSAInput::set_base_params(const char *device_name, snd_pcm_t *pcm_handle, snd_pcm_hw_params_t *hw_params, unsigned *sample_rate) +{ + int err; + err = snd_pcm_hw_params_any(pcm_handle, hw_params); + if (err < 0) { + fprintf(stderr, "[%s] snd_pcm_hw_params_any(): %s\n", device_name, snd_strerror(err)); + return false; + } + err = snd_pcm_hw_params_set_access(pcm_handle, hw_params, SND_PCM_ACCESS_RW_INTERLEAVED); + if (err < 0) { + fprintf(stderr, "[%s] snd_pcm_hw_params_set_access(): %s\n", device_name, snd_strerror(err)); + return false; + } + snd_pcm_format_mask_t *format_mask; + snd_pcm_format_mask_alloca(&format_mask); + snd_pcm_format_mask_set(format_mask, SND_PCM_FORMAT_S16_LE); + snd_pcm_format_mask_set(format_mask, SND_PCM_FORMAT_S24_LE); + snd_pcm_format_mask_set(format_mask, SND_PCM_FORMAT_S32_LE); + err = snd_pcm_hw_params_set_format_mask(pcm_handle, hw_params, format_mask); + if (err < 0) { + fprintf(stderr, "[%s] snd_pcm_hw_params_set_format_mask(): %s\n", device_name, snd_strerror(err)); + return false; + } + err = snd_pcm_hw_params_set_rate_near(pcm_handle, hw_params, sample_rate, 0); + if (err < 0) { + fprintf(stderr, "[%s] snd_pcm_hw_params_set_rate_near(): %s\n", device_name, snd_strerror(err)); + return false; + } + return true; +} + +ALSAInput::~ALSAInput() +{ + if (pcm_handle) { + WARN_ON_ERROR("snd_pcm_close()", snd_pcm_close(pcm_handle)); + } +} + +void ALSAInput::start_capture_thread() +{ + assert(!device.empty()); + should_quit.unquit(); + capture_thread = thread(&ALSAInput::capture_thread_func, this); +} + +void ALSAInput::stop_capture_thread() +{ + should_quit.quit(); + capture_thread.join(); +} + +void ALSAInput::capture_thread_func() +{ + parent_pool->set_card_state(internal_dev_index, ALSAPool::Device::State::STARTING); + + // If the device hasn't been opened already, we need to do so + // before we can capture. + while (!should_quit.should_quit() && pcm_handle == nullptr) { + if (!open_device()) { + fprintf(stderr, "[%s] Waiting one second and trying again...\n", + device.c_str()); + should_quit.sleep_for(seconds(1)); + } + } + + if (should_quit.should_quit()) { + // Don't call free_card(); that would be a deadlock. + if (pcm_handle) { + WARN_ON_ERROR("snd_pcm_close()", snd_pcm_close(pcm_handle)); + } + pcm_handle = nullptr; + return; + } + + // Do the actual capture. (Termination condition within loop.) + for ( ;; ) { + switch (do_capture()) { + case CaptureEndReason::REQUESTED_QUIT: + // Don't call free_card(); that would be a deadlock. + WARN_ON_ERROR("snd_pcm_close()", snd_pcm_close(pcm_handle)); + pcm_handle = nullptr; + return; + case CaptureEndReason::DEVICE_GONE: + parent_pool->free_card(internal_dev_index); + WARN_ON_ERROR("snd_pcm_close()", snd_pcm_close(pcm_handle)); + pcm_handle = nullptr; + return; + case CaptureEndReason::OTHER_ERROR: + parent_pool->set_card_state(internal_dev_index, ALSAPool::Device::State::STARTING); + fprintf(stderr, "[%s] Sleeping one second and restarting capture...\n", + device.c_str()); + should_quit.sleep_for(seconds(1)); + break; + } + } +} + +ALSAInput::CaptureEndReason ALSAInput::do_capture() +{ + parent_pool->set_card_state(internal_dev_index, ALSAPool::Device::State::STARTING); + RETURN_ON_ERROR("snd_pcm_start()", snd_pcm_start(pcm_handle)); + parent_pool->set_card_state(internal_dev_index, ALSAPool::Device::State::RUNNING); + + uint64_t num_frames_output = 0; + while (!should_quit.should_quit()) { + int ret = snd_pcm_wait(pcm_handle, /*timeout=*/100); + if (ret == 0) continue; // Timeout. + if (ret == -EPIPE) { + fprintf(stderr, "[%s] ALSA overrun\n", device.c_str()); + snd_pcm_prepare(pcm_handle); + snd_pcm_start(pcm_handle); + continue; + } + RETURN_ON_ERROR("snd_pcm_wait()", ret); + + snd_pcm_sframes_t frames = snd_pcm_readi(pcm_handle, buffer.get(), buffer_frames); + if (frames == -EPIPE) { + fprintf(stderr, "[%s] ALSA overrun\n", device.c_str()); + snd_pcm_prepare(pcm_handle); + snd_pcm_start(pcm_handle); + continue; + } + if (frames == 0) { + fprintf(stderr, "snd_pcm_readi() returned 0\n"); + break; + } + RETURN_ON_ERROR("snd_pcm_readi()", frames); + + const int64_t prev_pts = frames_to_pts(num_frames_output); + const int64_t pts = frames_to_pts(num_frames_output + frames); + const steady_clock::time_point now = steady_clock::now(); + bool success; + do { + if (should_quit.should_quit()) return CaptureEndReason::REQUESTED_QUIT; + success = audio_callback(buffer.get(), frames, audio_format, pts - prev_pts, now); + } while (!success); + num_frames_output += frames; + } + return CaptureEndReason::REQUESTED_QUIT; +} + +int64_t ALSAInput::frames_to_pts(uint64_t n) const +{ + return (n * TIMEBASE) / sample_rate; +} + diff --git a/nageru/alsa_input.h b/nageru/alsa_input.h new file mode 100644 index 0000000..060b921 --- /dev/null +++ b/nageru/alsa_input.h @@ -0,0 +1,78 @@ +#ifndef _ALSA_INPUT_H +#define _ALSA_INPUT_H 1 + +// ALSA sound input, running in a separate thread and sending audio back +// in callbacks. +// +// Note: “frame” here generally refers to the ALSA definition of frame, +// which is a set of samples, exactly one for each channel. The only exception +// is in frame_length, where it means the TIMEBASE length of the buffer +// as a whole, since that's what AudioMixer::add_audio() wants. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bmusb/bmusb.h" +#include "quittable_sleeper.h" + +class ALSAPool; + +class ALSAInput { +public: + typedef std::function audio_callback_t; + + ALSAInput(const char *device, unsigned sample_rate, unsigned num_channels, audio_callback_t audio_callback, ALSAPool *parent_pool, unsigned internal_dev_index); + ~ALSAInput(); + + // If not called before start_capture_thread(), the capture thread + // will call it until it succeeds. + bool open_device(); + + // Not valid before the device has been successfully opened. + // NOTE: Might very well be different from the sample rate given to the + // constructor, since the card might not support the one you wanted. + unsigned get_sample_rate() const { return sample_rate; } + + void start_capture_thread(); + void stop_capture_thread(); + + // Set access, sample rate and format parameters on the given ALSA PCM handle. + // Returns the computed parameter set and the chosen sample rate. Note that + // sample_rate is an in/out parameter; you send in the desired rate, + // and ALSA picks one as close to that as possible. + static bool set_base_params(const char *device_name, snd_pcm_t *pcm_handle, snd_pcm_hw_params_t *hw_params, unsigned *sample_rate); + +private: + void capture_thread_func(); + int64_t frames_to_pts(uint64_t n) const; + + enum class CaptureEndReason { + REQUESTED_QUIT, + DEVICE_GONE, + OTHER_ERROR + }; + CaptureEndReason do_capture(); + + std::string device; + unsigned sample_rate, num_channels, num_periods; + snd_pcm_uframes_t period_size; + snd_pcm_uframes_t buffer_frames; + bmusb::AudioFormat audio_format; + audio_callback_t audio_callback; + + snd_pcm_t *pcm_handle = nullptr; + std::thread capture_thread; + QuittableSleeper should_quit; + std::unique_ptr buffer; + ALSAPool *parent_pool; + unsigned internal_dev_index; +}; + +#endif // !defined(_ALSA_INPUT_H) diff --git a/nageru/alsa_output.cpp b/nageru/alsa_output.cpp new file mode 100644 index 0000000..7dd1024 --- /dev/null +++ b/nageru/alsa_output.cpp @@ -0,0 +1,96 @@ +#include "alsa_output.h" + +#include +#include +#include +#include +#include + +using namespace std; + +namespace { + +void die_on_error(const char *func_name, int err) +{ + if (err < 0) { + fprintf(stderr, "%s: %s\n", func_name, snd_strerror(err)); + exit(1); + } +} + +} // namespace + +ALSAOutput::ALSAOutput(int sample_rate, int num_channels) + : sample_rate(sample_rate), num_channels(num_channels) +{ + die_on_error("snd_pcm_open()", snd_pcm_open(&pcm_handle, "default", SND_PCM_STREAM_PLAYBACK, 0)); + + // Set format. + snd_pcm_hw_params_t *hw_params; + snd_pcm_hw_params_alloca(&hw_params); + die_on_error("snd_pcm_hw_params_any()", snd_pcm_hw_params_any(pcm_handle, hw_params)); + die_on_error("snd_pcm_hw_params_set_access()", snd_pcm_hw_params_set_access(pcm_handle, hw_params, SND_PCM_ACCESS_RW_INTERLEAVED)); + die_on_error("snd_pcm_hw_params_set_format()", snd_pcm_hw_params_set_format(pcm_handle, hw_params, SND_PCM_FORMAT_FLOAT_LE)); + die_on_error("snd_pcm_hw_params_set_rate()", snd_pcm_hw_params_set_rate(pcm_handle, hw_params, sample_rate, 0)); + die_on_error("snd_pcm_hw_params_set_channels", snd_pcm_hw_params_set_channels(pcm_handle, hw_params, num_channels)); + + // Fragment size of 512 samples. (A frame at 60 fps/48 kHz is 800 samples.) + // We ask for 16 such periods (~170 ms buffer). + unsigned int num_periods = 16; + int dir = 0; + die_on_error("snd_pcm_hw_params_set_periods_near()", snd_pcm_hw_params_set_periods_near(pcm_handle, hw_params, &num_periods, &dir)); + period_size = 512; + dir = 0; + die_on_error("snd_pcm_hw_params_set_period_size_near()", snd_pcm_hw_params_set_period_size_near(pcm_handle, hw_params, &period_size, &dir)); + die_on_error("snd_pcm_hw_params()", snd_pcm_hw_params(pcm_handle, hw_params)); + //snd_pcm_hw_params_free(hw_params); + + snd_pcm_sw_params_t *sw_params; + snd_pcm_sw_params_alloca(&sw_params); + die_on_error("snd_pcm_sw_params_current()", snd_pcm_sw_params_current(pcm_handle, sw_params)); + die_on_error("snd_pcm_sw_params_set_start_threshold", snd_pcm_sw_params_set_start_threshold(pcm_handle, sw_params, num_periods * period_size / 2)); + die_on_error("snd_pcm_sw_params()", snd_pcm_sw_params(pcm_handle, sw_params)); + + die_on_error("snd_pcm_nonblock", snd_pcm_nonblock(pcm_handle, 1)); + die_on_error("snd_pcm_prepare()", snd_pcm_prepare(pcm_handle)); +} + +void ALSAOutput::write(const vector &samples) +{ + buffer.insert(buffer.end(), samples.begin(), samples.end()); + +try_again: + int periods_to_write = buffer.size() / (period_size * num_channels); + if (periods_to_write == 0) { + return; + } + + int ret = snd_pcm_writei(pcm_handle, buffer.data(), periods_to_write * period_size); + if (ret == -EPIPE) { + fprintf(stderr, "warning: snd_pcm_writei() reported underrun\n"); + snd_pcm_recover(pcm_handle, ret, 1); + goto try_again; + } else if (ret == -EAGAIN) { + ret = 0; + } else if (ret < 0) { + fprintf(stderr, "error: snd_pcm_writei() returned '%s'\n", snd_strerror(ret)); + exit(1); + } else if (ret > 0) { + buffer.erase(buffer.begin(), buffer.begin() + ret * num_channels); + } + + if (buffer.size() >= period_size * num_channels) { // Still more to write. + if (ret == 0) { + if (buffer.size() >= period_size * num_channels * 8) { + // OK, almost 100 ms. Giving up. + fprintf(stderr, "warning: ALSA overrun, dropping some audio (%d ms)\n", + int(buffer.size() * 1000 / (num_channels * sample_rate))); + buffer.clear(); + } + } else if (ret > 0) { + // Not a completely failure (effectively a short write), + // possibly due to a signal. + goto try_again; + } + } +} diff --git a/nageru/alsa_output.h b/nageru/alsa_output.h new file mode 100644 index 0000000..3d1d2ca --- /dev/null +++ b/nageru/alsa_output.h @@ -0,0 +1,26 @@ +#ifndef _ALSA_OUTPUT_H +#define _ALSA_OUTPUT_H 1 + +// Extremely minimalistic ALSA output. Will not resample to fit +// sound card clock, will not care much about over- or underflows +// (so it will not block), will not care about A/V sync. +// +// This means that if you run it for long enough, clocks will +// probably drift out of sync enough to make a little pop. + +#include +#include + +class ALSAOutput { +public: + ALSAOutput(int sample_rate, int num_channels); + void write(const std::vector &samples); + +private: + snd_pcm_t *pcm_handle; + std::vector buffer; + snd_pcm_uframes_t period_size; + int sample_rate, num_channels; +}; + +#endif // !defined(_ALSA_OUTPUT_H) diff --git a/nageru/alsa_pool.cpp b/nageru/alsa_pool.cpp new file mode 100644 index 0000000..3092dc3 --- /dev/null +++ b/nageru/alsa_pool.cpp @@ -0,0 +1,547 @@ +#include "alsa_pool.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "alsa_input.h" +#include "audio_mixer.h" +#include "defs.h" +#include "input_mapping.h" +#include "state.pb.h" + +using namespace std; +using namespace std::placeholders; + +ALSAPool::ALSAPool() +{ + should_quit_fd = eventfd(/*initval=*/0, /*flags=*/0); + assert(should_quit_fd != -1); +} + +ALSAPool::~ALSAPool() +{ + for (Device &device : devices) { + if (device.input != nullptr) { + device.input->stop_capture_thread(); + } + } + should_quit = true; + const uint64_t one = 1; + if (write(should_quit_fd, &one, sizeof(one)) != sizeof(one)) { + perror("write(should_quit_fd)"); + exit(1); + } + inotify_thread.join(); + + while (retry_threads_running > 0) { + this_thread::sleep_for(std::chrono::milliseconds(100)); + } +} + +std::vector ALSAPool::get_devices() +{ + lock_guard lock(mu); + for (Device &device : devices) { + device.held = true; + } + return devices; +} + +void ALSAPool::hold_device(unsigned index) +{ + lock_guard lock(mu); + assert(index < devices.size()); + devices[index].held = true; +} + +void ALSAPool::release_device(unsigned index) +{ + lock_guard lock(mu); + if (index < devices.size()) { + devices[index].held = false; + } +} + +void ALSAPool::enumerate_devices() +{ + // Enumerate all cards. + for (int card_index = -1; snd_card_next(&card_index) == 0 && card_index >= 0; ) { + char address[256]; + snprintf(address, sizeof(address), "hw:%d", card_index); + + snd_ctl_t *ctl; + int err = snd_ctl_open(&ctl, address, 0); + if (err < 0) { + printf("%s: %s\n", address, snd_strerror(err)); + continue; + } + unique_ptr ctl_closer(ctl, snd_ctl_close); + + // Enumerate all devices on this card. + for (int dev_index = -1; snd_ctl_pcm_next_device(ctl, &dev_index) == 0 && dev_index >= 0; ) { + probe_device_with_retry(card_index, dev_index); + } + } +} + +void ALSAPool::probe_device_with_retry(unsigned card_index, unsigned dev_index) +{ + char address[256]; + snprintf(address, sizeof(address), "hw:%d,%d", card_index, dev_index); + + lock_guard lock(add_device_mutex); + if (add_device_tries_left.count(address)) { + // Some thread is already busy retrying this, + // so just reset its count. + add_device_tries_left[address] = num_retries; + return; + } + + // Try (while still holding the lock) to add the device synchronously. + ProbeResult result = probe_device_once(card_index, dev_index); + if (result == ProbeResult::SUCCESS) { + return; + } else if (result == ProbeResult::FAILURE) { + return; + } + assert(result == ProbeResult::DEFER); + + // Add failed for whatever reason (probably just that the device + // isn't up yet. Set up a count so that nobody else starts a thread, + // then start it ourselves. + fprintf(stderr, "Trying %s again in one second...\n", address); + add_device_tries_left[address] = num_retries; + ++retry_threads_running; + thread(&ALSAPool::probe_device_retry_thread_func, this, card_index, dev_index).detach(); +} + +void ALSAPool::probe_device_retry_thread_func(unsigned card_index, unsigned dev_index) +{ + char address[256]; + snprintf(address, sizeof(address), "hw:%d,%d", card_index, dev_index); + + char thread_name[16]; + snprintf(thread_name, sizeof(thread_name), "Reprobe_hw:%d,%d", card_index, dev_index); + pthread_setname_np(pthread_self(), thread_name); + + for ( ;; ) { // Termination condition within the loop. + sleep(1); + + // See if there are any retries left. + lock_guard lock(add_device_mutex); + if (should_quit || + !add_device_tries_left.count(address) || + add_device_tries_left[address] == 0) { + add_device_tries_left.erase(address); + fprintf(stderr, "Giving up probe of %s.\n", address); + break; + } + + // Seemingly there were. Give it a try (we still hold the mutex). + ProbeResult result = probe_device_once(card_index, dev_index); + if (result == ProbeResult::SUCCESS) { + add_device_tries_left.erase(address); + fprintf(stderr, "Probe of %s succeeded.\n", address); + break; + } else if (result == ProbeResult::FAILURE || --add_device_tries_left[address] == 0) { + add_device_tries_left.erase(address); + fprintf(stderr, "Giving up probe of %s.\n", address); + break; + } + + // Failed again. + assert(result == ProbeResult::DEFER); + fprintf(stderr, "Trying %s again in one second (%d tries left)...\n", + address, add_device_tries_left[address]); + } + + --retry_threads_running; +} + +ALSAPool::ProbeResult ALSAPool::probe_device_once(unsigned card_index, unsigned dev_index) +{ + char address[256]; + snprintf(address, sizeof(address), "hw:%d", card_index); + snd_ctl_t *ctl; + int err = snd_ctl_open(&ctl, address, 0); + if (err < 0) { + printf("%s: %s\n", address, snd_strerror(err)); + return ALSAPool::ProbeResult::DEFER; + } + unique_ptr ctl_closer(ctl, snd_ctl_close); + + snprintf(address, sizeof(address), "hw:%d,%d", card_index, dev_index); + + snd_pcm_info_t *pcm_info; + snd_pcm_info_alloca(&pcm_info); + snd_pcm_info_set_device(pcm_info, dev_index); + snd_pcm_info_set_subdevice(pcm_info, 0); + snd_pcm_info_set_stream(pcm_info, SND_PCM_STREAM_CAPTURE); + err = snd_ctl_pcm_info(ctl, pcm_info); + if (err == -ENOENT) { + // Not a capture card. + return ALSAPool::ProbeResult::FAILURE; + } + if (err < 0) { + // Not available for capture. + printf("%s: Not available for capture.\n", address); + return ALSAPool::ProbeResult::DEFER; + } + + unsigned num_channels = 0; + + // Find all channel maps for this device, and pick out the one + // with the most channels. + snd_pcm_chmap_query_t **cmaps = snd_pcm_query_chmaps_from_hw(card_index, dev_index, 0, SND_PCM_STREAM_CAPTURE); + if (cmaps != nullptr) { + for (snd_pcm_chmap_query_t **ptr = cmaps; *ptr; ++ptr) { + num_channels = max(num_channels, (*ptr)->map.channels); + } + snd_pcm_free_chmaps(cmaps); + } + if (num_channels == 0) { + // Device had no channel maps. We need to open it to query. + // TODO: Do this asynchronously. + snd_pcm_t *pcm_handle; + int err = snd_pcm_open(&pcm_handle, address, SND_PCM_STREAM_CAPTURE, 0); + if (err < 0) { + printf("%s: %s\n", address, snd_strerror(err)); + return ALSAPool::ProbeResult::DEFER; + } + snd_pcm_hw_params_t *hw_params; + snd_pcm_hw_params_alloca(&hw_params); + unsigned sample_rate; + if (!ALSAInput::set_base_params(address, pcm_handle, hw_params, &sample_rate)) { + snd_pcm_close(pcm_handle); + return ALSAPool::ProbeResult::DEFER; + } + err = snd_pcm_hw_params_get_channels_max(hw_params, &num_channels); + if (err < 0) { + fprintf(stderr, "[%s] snd_pcm_hw_params_get_channels_max(): %s\n", + address, snd_strerror(err)); + snd_pcm_close(pcm_handle); + return ALSAPool::ProbeResult::DEFER; + } + snd_pcm_close(pcm_handle); + } + + if (num_channels == 0) { + printf("%s: No channel maps with channels\n", address); + return ALSAPool::ProbeResult::FAILURE; + } + + snd_ctl_card_info_t *card_info; + snd_ctl_card_info_alloca(&card_info); + snd_ctl_card_info(ctl, card_info); + + string name = snd_ctl_card_info_get_name(card_info); + string info = snd_pcm_info_get_name(pcm_info); + + unsigned internal_dev_index; + string display_name; + { + lock_guard lock(mu); + internal_dev_index = find_free_device_index(name, info, num_channels, address); + devices[internal_dev_index].address = address; + devices[internal_dev_index].name = name; + devices[internal_dev_index].info = info; + devices[internal_dev_index].num_channels = num_channels; + // Note: Purposefully does not overwrite held. + + display_name = devices[internal_dev_index].display_name(); + } + + fprintf(stderr, "%s: Probed successfully.\n", address); + + reset_device(internal_dev_index); // Restarts it if it is held (ie., we just replaced a dead card). + + DeviceSpec spec{InputSourceType::ALSA_INPUT, internal_dev_index}; + global_audio_mixer->set_display_name(spec, display_name); + global_audio_mixer->trigger_state_changed_callback(); + + return ALSAPool::ProbeResult::SUCCESS; +} + +void ALSAPool::unplug_device(unsigned card_index, unsigned dev_index) +{ + char address[256]; + snprintf(address, sizeof(address), "hw:%d,%d", card_index, dev_index); + for (unsigned i = 0; i < devices.size(); ++i) { + if (devices[i].state != Device::State::EMPTY && + devices[i].state != Device::State::DEAD && + devices[i].address == address) { + free_card(i); + } + } +} + +void ALSAPool::init() +{ + inotify_thread = thread(&ALSAPool::inotify_thread_func, this); + enumerate_devices(); +} + +void ALSAPool::inotify_thread_func() +{ + pthread_setname_np(pthread_self(), "ALSA_Hotplug"); + + int inotify_fd = inotify_init(); + if (inotify_fd == -1) { + perror("inotify_init()"); + fprintf(stderr, "No hotplug of ALSA devices available.\n"); + return; + } + + int watch_fd = inotify_add_watch(inotify_fd, "/dev/snd", IN_MOVE | IN_CREATE | IN_DELETE); + if (watch_fd == -1) { + perror("inotify_add_watch()"); + fprintf(stderr, "No hotplug of ALSA devices available.\n"); + close(inotify_fd); + return; + } + + int size = sizeof(inotify_event) + NAME_MAX + 1; + unique_ptr buf(new char[size]); + while (!should_quit) { + pollfd fds[2]; + fds[0].fd = inotify_fd; + fds[0].events = POLLIN; + fds[0].revents = 0; + fds[1].fd = should_quit_fd; + fds[1].events = POLLIN; + fds[1].revents = 0; + + int ret = poll(fds, 2, -1); + if (ret == -1) { + if (errno == EINTR) { + continue; + } else { + perror("poll(inotify_fd)"); + return; + } + } + if (ret == 0) { + continue; + } + + if (fds[1].revents) break; // should_quit_fd asserted. + + ret = read(inotify_fd, buf.get(), size); + if (ret == -1) { + if (errno == EINTR) { + continue; + } else { + perror("read(inotify_fd)"); + close(watch_fd); + close(inotify_fd); + return; + } + } + if (ret < int(sizeof(inotify_event))) { + fprintf(stderr, "inotify read unexpectedly returned %d, giving up hotplug of ALSA devices.\n", + int(ret)); + close(watch_fd); + close(inotify_fd); + return; + } + + for (int i = 0; i < ret; ) { + const inotify_event *event = reinterpret_cast(&buf[i]); + i += sizeof(inotify_event) + event->len; + + if (event->mask & IN_Q_OVERFLOW) { + fprintf(stderr, "WARNING: inotify overflowed, may lose ALSA hotplug events.\n"); + continue; + } + unsigned card, device; + char type; + if (sscanf(event->name, "pcmC%uD%u%c", &card, &device, &type) == 3 && type == 'c') { + if (event->mask & (IN_MOVED_FROM | IN_DELETE)) { + printf("Deleted capture device: Card %u, device %u\n", card, device); + unplug_device(card, device); + } + if (event->mask & (IN_MOVED_TO | IN_CREATE)) { + printf("Adding capture device: Card %u, device %u\n", card, device); + probe_device_with_retry(card, device); + } + } + } + } + close(watch_fd); + close(inotify_fd); + close(should_quit_fd); +} + +void ALSAPool::reset_device(unsigned index) +{ + lock_guard lock(mu); + Device *device = &devices[index]; + if (device->state == Device::State::DEAD) { + // Not running, and should not be started. + return; + } + if (inputs[index] != nullptr) { + inputs[index]->stop_capture_thread(); + } + if (!device->held) { + inputs[index].reset(); + } else { + // TODO: Put on a background thread instead of locking? + auto callback = bind(&AudioMixer::add_audio, global_audio_mixer, DeviceSpec{InputSourceType::ALSA_INPUT, index}, _1, _2, _3, _4, _5); + inputs[index].reset(new ALSAInput(device->address.c_str(), OUTPUT_FREQUENCY, device->num_channels, callback, this, index)); + inputs[index]->start_capture_thread(); + } + device->input = inputs[index].get(); +} + +unsigned ALSAPool::get_capture_frequency(unsigned index) +{ + lock_guard lock(mu); + assert(devices[index].held); + if (devices[index].input) + return devices[index].input->get_sample_rate(); + else + return OUTPUT_FREQUENCY; +} + +ALSAPool::Device::State ALSAPool::get_card_state(unsigned index) +{ + lock_guard lock(mu); + assert(devices[index].held); + return devices[index].state; +} + +void ALSAPool::set_card_state(unsigned index, ALSAPool::Device::State state) +{ + { + lock_guard lock(mu); + devices[index].state = state; + } + + DeviceSpec spec{InputSourceType::ALSA_INPUT, index}; + bool silence = (state != ALSAPool::Device::State::RUNNING); + while (!global_audio_mixer->silence_card(spec, silence)) + ; + global_audio_mixer->trigger_state_changed_callback(); +} + +unsigned ALSAPool::find_free_device_index(const string &name, const string &info, unsigned num_channels, const string &address) +{ + // First try to find an exact match on a dead card. + for (unsigned i = 0; i < devices.size(); ++i) { + if (devices[i].state == Device::State::DEAD && + devices[i].address == address && + devices[i].name == name && + devices[i].info == info && + devices[i].num_channels == num_channels) { + devices[i].state = Device::State::READY; + return i; + } + } + + // Then try to find a match on everything but the address + // (probably that devices were plugged back in a different order). + // If we have two cards that are equal, this might get them mixed up, + // but we don't have anything better. + for (unsigned i = 0; i < devices.size(); ++i) { + if (devices[i].state == Device::State::DEAD && + devices[i].name == name && + devices[i].info == info && + devices[i].num_channels == num_channels) { + devices[i].state = Device::State::READY; + return i; + } + } + + // OK, so we didn't find a match; see if there are any empty slots. + for (unsigned i = 0; i < devices.size(); ++i) { + if (devices[i].state == Device::State::EMPTY) { + devices[i].state = Device::State::READY; + devices[i].held = false; + return i; + } + } + + // Failing that, we just insert the new device at the end. + Device new_dev; + new_dev.state = Device::State::READY; + new_dev.held = false; + devices.push_back(new_dev); + inputs.emplace_back(nullptr); + return devices.size() - 1; +} + +unsigned ALSAPool::create_dead_card(const string &name, const string &info, unsigned num_channels) +{ + lock_guard lock(mu); + + // See if there are any empty slots. If not, insert one at the end. + vector::iterator free_device = + find_if(devices.begin(), devices.end(), + [](const Device &device) { return device.state == Device::State::EMPTY; }); + if (free_device == devices.end()) { + devices.push_back(Device()); + inputs.emplace_back(nullptr); + free_device = devices.end() - 1; + } + + free_device->state = Device::State::DEAD; + free_device->name = name; + free_device->info = info; + free_device->num_channels = num_channels; + free_device->held = true; + + return distance(devices.begin(), free_device); +} + +void ALSAPool::serialize_device(unsigned index, DeviceSpecProto *serialized) +{ + lock_guard lock(mu); + assert(index < devices.size()); + assert(devices[index].held); + serialized->set_type(DeviceSpecProto::ALSA_INPUT); + serialized->set_index(index); + serialized->set_display_name(devices[index].display_name()); + serialized->set_alsa_name(devices[index].name); + serialized->set_alsa_info(devices[index].info); + serialized->set_num_channels(devices[index].num_channels); + serialized->set_address(devices[index].address); +} + +void ALSAPool::free_card(unsigned index) +{ + DeviceSpec spec{InputSourceType::ALSA_INPUT, index}; + while (!global_audio_mixer->silence_card(spec, true)) + ; + + { + lock_guard lock(mu); + if (devices[index].held) { + devices[index].state = Device::State::DEAD; + } else { + devices[index].state = Device::State::EMPTY; + inputs[index].reset(); + } + while (!devices.empty() && devices.back().state == Device::State::EMPTY) { + devices.pop_back(); + inputs.pop_back(); + } + } + + global_audio_mixer->trigger_state_changed_callback(); +} diff --git a/nageru/alsa_pool.h b/nageru/alsa_pool.h new file mode 100644 index 0000000..904e2ec --- /dev/null +++ b/nageru/alsa_pool.h @@ -0,0 +1,155 @@ +#ifndef _ALSA_POOL_H +#define _ALSA_POOL_H 1 + +#include +#include +#include +#include +#include +#include +#include + +class ALSAInput; +class DeviceSpecProto; + +// The class dealing with the collective of all ALSA cards in the system. +// In particular, it deals with enumeration of cards, and hotplug of new ones. +class ALSAPool { +public: + ALSAPool(); + ~ALSAPool(); + + struct Device { + enum class State { + // There is no card here. (There probably used to be one, + // but it got removed.) We don't insert a card before + // we've actually probed it, ie., we know whether it + // can be captured from at all, and what its name is. + EMPTY, + + // This card is ready for capture, as far as we know. + // (It could still be used by someone else; we don't know + // until we try to open it.) + READY, + + // We are trying to start capture from this card, but we are not + // streaming yet. Note that this could in theory go on forever, + // if the card is in use by some other process; in the UI, + // we will show this state as “(busy)”. + STARTING, + + // The card is capturing and sending data. If there's a fatal error, + // it could go back to STARTING, or it could go to DEAD + // (depending on the error). + RUNNING, + + // The card is gone (e.g., unplugged). However, since there's + // still a bus using it, we can't just remove the entry. + // If the card comes back (ie., a new card is plugged in, + // and we believe it has the same configuration), it could be + // installed in place of this card, and then presumably be put + // back into STARTING or RUNNING. + DEAD + } state = State::EMPTY; + + std::string address; // E.g. “hw:0,0”. + std::string name, info; + unsigned num_channels; + ALSAInput *input = nullptr; // nullptr iff EMPTY or DEAD. + + // Whether the AudioMixer is interested in this card or not. + // “Interested” could mean either of two things: Either it is part of + // a bus mapping, or it is in the process of enumerating devices + // (to show to the user). A card that is _not_ held can disappear + // at any given time as a result of an error or hotplug event; + // a card that is held will go to the DEAD state instead. + bool held = false; + + std::string display_name() const { return name + " (" + info + ")"; } + }; + + void init(); + + // Get the list of all current devices. Note that this will implicitly mark + // all of the returned devices as held, since the input mapping UI needs + // some kind of stability when the user is to choose. Thus, when you are done + // with the list and have set a new mapping, you must go through all the devices + // you don't want and release them using release_device(). + std::vector get_devices(); + + void hold_device(unsigned index); + void release_device(unsigned index); // Note: index is allowed to go out of bounds. + + // If device is held, start or restart capture. If device is not held, + // stop capture if it isn't already. + void reset_device(unsigned index); + + // Note: The card must be held. Returns OUTPUT_FREQUENCY if the card is in EMPTY or DEAD. + unsigned get_capture_frequency(unsigned index); + + // Note: The card must be held. + Device::State get_card_state(unsigned index); + + // Only for ALSAInput. + void set_card_state(unsigned index, Device::State state); + + // Just a short form for taking and then moving the card to + // EMPTY or DEAD state. Only for ALSAInput and for internal use. + void free_card(unsigned index); + + // Create a new card, mark it immediately as DEAD and hold it. + // Returns the new index. + unsigned create_dead_card(const std::string &name, const std::string &info, unsigned num_channels); + + // Make a protobuf representation of the given card, so that it can be + // matched against at a later stage. For AudioMixer only. + // The given card must be held. + void serialize_device(unsigned index, DeviceSpecProto *serialized); + +private: + mutable std::mutex mu; + std::vector devices; // Under mu. + std::vector> inputs; // Under mu, corresponds 1:1 to devices. + + // Keyed on device address (e.g. “hw:0,0”). If there's an entry here, + // it means we already have a thread doing retries, so we shouldn't + // start a new one. + std::unordered_map add_device_tries_left; // Under add_device_mutex. + std::mutex add_device_mutex; + + static constexpr int num_retries = 10; + + void inotify_thread_func(); + void enumerate_devices(); + + // Try to add an input at the given card/device. If it succeeds, return + // synchronously. If not, fire off a background thread to try up to + // times. + void probe_device_with_retry(unsigned card_index, unsigned dev_index); + void probe_device_retry_thread_func(unsigned card_index, unsigned dev_index); + + enum class ProbeResult { + SUCCESS, + DEFER, + FAILURE + }; + ProbeResult probe_device_once(unsigned card_index, unsigned dev_index); + + void unplug_device(unsigned card_index, unsigned dev_index); + + // Must be called with held. Will allocate a new entry if needed. + // The returned entry will be set to READY state. + unsigned find_free_device_index(const std::string &name, + const std::string &info, + unsigned num_channels, + const std::string &address); + + std::atomic should_quit{false}; + int should_quit_fd; + std::thread inotify_thread; + std::atomic retry_threads_running{0}; + + friend class ALSAInput; +}; + +#endif // !defined(_ALSA_POOL_H) diff --git a/nageru/analyzer.cpp b/nageru/analyzer.cpp new file mode 100644 index 0000000..b24b46a --- /dev/null +++ b/nageru/analyzer.cpp @@ -0,0 +1,394 @@ +#include "analyzer.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include "context.h" +#include "flags.h" +#include "mixer.h" +#include "ui_analyzer.h" + +// QCustomPlot includes qopenglfunctions.h, which #undefs all of the epoxy +// definitions (ugh) and doesn't put back any others (ugh). Add the ones we +// need back. + +#define glBindBuffer epoxy_glBindBuffer +#define glBindFramebuffer epoxy_glBindFramebuffer +#define glBufferData epoxy_glBufferData +#define glDeleteBuffers epoxy_glDeleteBuffers +#define glDisable epoxy_glDisable +#define glGenBuffers epoxy_glGenBuffers +#define glGetError epoxy_glGetError +#define glReadPixels epoxy_glReadPixels +#define glUnmapBuffer epoxy_glUnmapBuffer +#define glWaitSync epoxy_glWaitSync + +using namespace std; + +Analyzer::Analyzer() + : ui(new Ui::Analyzer), + grabbed_image(global_flags.width, global_flags.height, QImage::Format_ARGB32_Premultiplied) +{ + ui->setupUi(this); + + surface = create_surface(QSurfaceFormat::defaultFormat()); + context = create_context(surface); + if (!make_current(context, surface)) { + printf("oops\n"); + exit(1); + } + + grab_timer.setSingleShot(true); + connect(&grab_timer, &QTimer::timeout, bind(&Analyzer::grab_clicked, this)); + + ui->input_box->addItem("Live", Mixer::OUTPUT_LIVE); + ui->input_box->addItem("Preview", Mixer::OUTPUT_PREVIEW); + unsigned num_channels = global_mixer->get_num_channels(); + for (unsigned channel_idx = 0; channel_idx < num_channels; ++channel_idx) { + Mixer::Output channel = static_cast(Mixer::OUTPUT_INPUT0 + channel_idx); + string name = global_mixer->get_channel_name(channel); + ui->input_box->addItem(QString::fromStdString(name), channel); + } + + ui->grab_frequency_box->addItem("Never", 0); + ui->grab_frequency_box->addItem("100 ms", 100); + ui->grab_frequency_box->addItem("1 sec", 1000); + ui->grab_frequency_box->addItem("10 sec", 10000); + ui->grab_frequency_box->setCurrentIndex(2); + + connect(ui->grab_btn, &QPushButton::clicked, bind(&Analyzer::grab_clicked, this)); + connect(ui->input_box, static_cast(&QComboBox::currentIndexChanged), bind(&Analyzer::signal_changed, this)); + signal_changed(); + ui->grabbed_frame_label->installEventFilter(this); + + glGenBuffers(1, &pbo); + glBindBuffer(GL_PIXEL_PACK_BUFFER_ARB, pbo); + glBufferData(GL_PIXEL_PACK_BUFFER_ARB, global_flags.width * global_flags.height * 4, nullptr, GL_STREAM_READ); + + ui->histogram->xAxis->setVisible(true); + ui->histogram->yAxis->setVisible(false); + ui->histogram->xAxis->setRange(0, 255); +} + +Analyzer::~Analyzer() +{ + delete_context(context); + delete surface; +} + +void Analyzer::update_channel_name(Mixer::Output output, const string &name) +{ + if (output >= Mixer::OUTPUT_INPUT0) { + int index = (output - Mixer::OUTPUT_INPUT0) + 2; + ui->input_box->setItemText(index, QString::fromStdString(name)); + } +} + +void Analyzer::mixer_shutting_down() +{ + ui->display->shutdown(); + + if (!make_current(context, surface)) { + printf("oops\n"); + exit(1); + } + glDeleteBuffers(1, &pbo); + check_error(); + if (resource_pool != nullptr) { + resource_pool->clean_context(); + } +} + +void Analyzer::grab_clicked() +{ + Mixer::Output channel = static_cast(ui->input_box->currentData().value()); + + if (!make_current(context, surface)) { + printf("oops\n"); + exit(1); + } + + Mixer::DisplayFrame frame; + if (!global_mixer->get_display_frame(channel, &frame)) { + // Not ready yet. + return; + } + + // Set up an FBO to render into. + if (resource_pool == nullptr) { + resource_pool = frame.chain->get_resource_pool(); + } else { + assert(resource_pool == frame.chain->get_resource_pool()); + } + GLuint fbo_tex = resource_pool->create_2d_texture(GL_RGBA8, global_flags.width, global_flags.height); + check_error(); + GLuint fbo = resource_pool->create_fbo(fbo_tex); + check_error(); + + glWaitSync(frame.ready_fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED); + check_error(); + frame.setup_chain(); + check_error(); + glDisable(GL_FRAMEBUFFER_SRGB); + check_error(); + frame.chain->render_to_fbo(fbo, global_flags.width, global_flags.height); + check_error(); + + // Read back to memory. + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + check_error(); + glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo); + check_error(); + glReadPixels(0, 0, global_flags.width, global_flags.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, BUFFER_OFFSET(0)); + check_error(); + + unsigned char *buf = (unsigned char *)glMapBuffer(GL_PIXEL_PACK_BUFFER, GL_READ_ONLY); + check_error(); + + size_t pitch = global_flags.width * 4; + for (int y = 0; y < global_flags.height; ++y) { + memcpy(grabbed_image.scanLine(global_flags.height - y - 1), buf + y * pitch, pitch); + } + + { + char buf[256]; + snprintf(buf, sizeof(buf), "Grabbed frame (%dx%d)", global_flags.width, global_flags.height); + ui->grabbed_frame_sublabel->setText(buf); + } + + QPixmap pixmap; + pixmap.convertFromImage(grabbed_image); + ui->grabbed_frame_label->setPixmap(pixmap); + + int r_hist[256] = {0}, g_hist[256] = {0}, b_hist[256] = {0}; + const unsigned char *ptr = buf; + for (int i = 0; i < global_flags.height * global_flags.width; ++i) { + uint8_t b = *ptr++; + uint8_t g = *ptr++; + uint8_t r = *ptr++; + ++ptr; + + ++r_hist[r]; + ++g_hist[g]; + ++b_hist[b]; + } + + glUnmapBuffer(GL_PIXEL_PACK_BUFFER); + check_error(); + glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); + check_error(); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + check_error(); + + QVector r_vec(256), g_vec(256), b_vec(256), x_vec(256); + double max = 0.0; + for (unsigned i = 0; i < 256; ++i) { + x_vec[i] = i; + r_vec[i] = log(r_hist[i] + 1.0); + g_vec[i] = log(g_hist[i] + 1.0); + b_vec[i] = log(b_hist[i] + 1.0); + + max = std::max(max, r_vec[i]); + max = std::max(max, g_vec[i]); + max = std::max(max, b_vec[i]); + } + + ui->histogram->clearGraphs(); + ui->histogram->addGraph(); + ui->histogram->graph(0)->setData(x_vec, r_vec); + ui->histogram->graph(0)->setPen(QPen(Qt::red)); + ui->histogram->graph(0)->setBrush(QBrush(QColor(255, 127, 127, 80))); + ui->histogram->addGraph(); + ui->histogram->graph(1)->setData(x_vec, g_vec); + ui->histogram->graph(1)->setPen(QPen(Qt::green)); + ui->histogram->graph(1)->setBrush(QBrush(QColor(127, 255, 127, 80))); + ui->histogram->addGraph(); + ui->histogram->graph(2)->setData(x_vec, b_vec); + ui->histogram->graph(2)->setPen(QPen(Qt::blue)); + ui->histogram->graph(2)->setBrush(QBrush(QColor(127, 127, 255, 80))); + + ui->histogram->xAxis->setVisible(true); + ui->histogram->yAxis->setVisible(false); + ui->histogram->xAxis->setRange(0, 255); + ui->histogram->yAxis->setRange(0, max); + ui->histogram->replot(); + + resource_pool->release_2d_texture(fbo_tex); + check_error(); + resource_pool->release_fbo(fbo); + check_error(); + + if (last_x >= 0 && last_y >= 0) { + grab_pixel(last_x, last_y); + } + + if (isVisible()) { + grab_timer.stop(); + + // Set up the next autograb if configured. + int delay = ui->grab_frequency_box->currentData().toInt(nullptr); + if (delay > 0) { + grab_timer.start(delay); + } + } +} + +void Analyzer::signal_changed() +{ + Mixer::Output channel = static_cast(ui->input_box->currentData().value()); + ui->display->set_output(channel); + grab_clicked(); +} + +bool Analyzer::eventFilter(QObject *watched, QEvent *event) +{ + if (event->type() == QEvent::MouseMove && watched->isWidgetType()) { + const QMouseEvent *mouse_event = (QMouseEvent *)event; + last_x = mouse_event->x(); + last_y = mouse_event->y(); + grab_pixel(mouse_event->x(), mouse_event->y()); + } + if (event->type() == QEvent::Leave && watched->isWidgetType()) { + last_x = last_y = -1; + ui->coord_label->setText("Selected coordinate (x,y): (none)"); + ui->red_label->setText(u8"—"); + ui->green_label->setText(u8"—"); + ui->blue_label->setText(u8"—"); + ui->hex_label->setText(u8"#—"); + } + return false; +} + +void Analyzer::grab_pixel(int x, int y) +{ + const QPixmap *pixmap = ui->grabbed_frame_label->pixmap(); + if (pixmap != nullptr) { + x = lrint(x * double(pixmap->width()) / ui->grabbed_frame_label->width()); + y = lrint(y * double(pixmap->height()) / ui->grabbed_frame_label->height()); + x = std::min(x, pixmap->width() - 1); + y = std::min(y, pixmap->height() - 1); + + char buf[256]; + snprintf(buf, sizeof(buf), "Selected coordinate (x,y): (%d,%d)", x, y); + ui->coord_label->setText(buf); + + QRgb pixel = grabbed_image.pixel(x, y); + ui->red_label->setText(QString::fromStdString(to_string(qRed(pixel)))); + ui->green_label->setText(QString::fromStdString(to_string(qGreen(pixel)))); + ui->blue_label->setText(QString::fromStdString(to_string(qBlue(pixel)))); + + snprintf(buf, sizeof(buf), "#%02x%02x%02x", qRed(pixel), qGreen(pixel), qBlue(pixel)); + ui->hex_label->setText(buf); + } +} + +void Analyzer::resizeEvent(QResizeEvent* event) +{ + QMainWindow::resizeEvent(event); + + // Ask for a relayout, but only after the event loop is done doing relayout + // on everything else. + QMetaObject::invokeMethod(this, "relayout", Qt::QueuedConnection); +} + +void Analyzer::showEvent(QShowEvent *event) +{ + grab_clicked(); +} + +void Analyzer::relayout() +{ + double aspect = double(global_flags.width) / global_flags.height; + + // Left pane (2/5 of the width). + { + int width = ui->left_pane->geometry().width(); + int height = ui->left_pane->geometry().height(); + + // Figure out how much space everything that's non-responsive needs. + int remaining_height = height - ui->left_pane->spacing() * (ui->left_pane->count() - 1); + + remaining_height -= ui->input_box->geometry().height(); + ui->left_pane->setStretch(2, ui->grab_btn->geometry().height()); + + remaining_height -= ui->grab_btn->geometry().height(); + ui->left_pane->setStretch(3, ui->grab_btn->geometry().height()); + + remaining_height -= ui->histogram_label->geometry().height(); + ui->left_pane->setStretch(5, ui->histogram_label->geometry().height()); + + // The histogram's minimumHeight returns 0, so let's set a reasonable minimum for it. + int min_histogram_height = 50; + remaining_height -= min_histogram_height; + + // Allocate so that the display is 16:9, if possible. + unsigned wanted_display_height = width / aspect; + unsigned display_height; + unsigned margin = 0; + if (remaining_height >= int(wanted_display_height)) { + display_height = wanted_display_height; + } else { + display_height = remaining_height; + int display_width = lrint(display_height * aspect); + margin = (width - display_width) / 2; + } + ui->left_pane->setStretch(1, display_height); + ui->display_left_spacer->changeSize(margin, 1); + ui->display_right_spacer->changeSize(margin, 1); + + remaining_height -= display_height; + + // Figure out if we can do the histogram at 16:9. + remaining_height += min_histogram_height; + unsigned histogram_height; + if (remaining_height >= int(wanted_display_height)) { + histogram_height = wanted_display_height; + } else { + histogram_height = remaining_height; + } + remaining_height -= histogram_height; + ui->left_pane->setStretch(4, histogram_height); + + ui->left_pane->setStretch(0, remaining_height / 2); + ui->left_pane->setStretch(6, remaining_height / 2); + } + + // Right pane (remaining 3/5 of the width). + { + int width = ui->right_pane->geometry().width(); + int height = ui->right_pane->geometry().height(); + + // Figure out how much space everything that's non-responsive needs. + int remaining_height = height - ui->right_pane->spacing() * (ui->right_pane->count() - 1); + remaining_height -= ui->grabbed_frame_sublabel->geometry().height(); + remaining_height -= ui->coord_label->geometry().height(); + remaining_height -= ui->color_hbox->geometry().height(); + + // Allocate so that the display is 16:9, if possible. + unsigned wanted_display_height = width / aspect; + unsigned display_height; + unsigned margin = 0; + if (remaining_height >= int(wanted_display_height)) { + display_height = wanted_display_height; + } else { + display_height = remaining_height; + int display_width = lrint(display_height * aspect); + margin = (width - display_width) / 2; + } + ui->right_pane->setStretch(1, display_height); + ui->grabbed_frame_left_spacer->changeSize(margin, 1); + ui->grabbed_frame_right_spacer->changeSize(margin, 1); + remaining_height -= display_height; + + if (remaining_height < 0) remaining_height = 0; + + ui->right_pane->setStretch(0, remaining_height / 2); + ui->right_pane->setStretch(5, remaining_height / 2); + } +} diff --git a/nageru/analyzer.h b/nageru/analyzer.h new file mode 100644 index 0000000..b5aad15 --- /dev/null +++ b/nageru/analyzer.h @@ -0,0 +1,58 @@ +#ifndef _ANALYZER_H +#define _ANALYZER_H 1 + +#include +#include +#include +#include + +#include + +#include + +#include "mixer.h" + +class QObject; +class QOpenGLContext; +class QSurface; + +namespace Ui { +class Analyzer; +} // namespace Ui + +namespace movit { +class ResourcePool; +} // namespace movit + +class Analyzer : public QMainWindow +{ + Q_OBJECT + +public: + Analyzer(); + ~Analyzer(); + void update_channel_name(Mixer::Output output, const std::string &name); + void mixer_shutting_down(); + +public slots: + void relayout(); + +private: + void grab_clicked(); + void signal_changed(); + void grab_pixel(int x, int y); + bool eventFilter(QObject *watched, QEvent *event) override; + void resizeEvent(QResizeEvent *event) override; + void showEvent(QShowEvent *event) override; + + Ui::Analyzer *ui; + QSurface *surface; + QOpenGLContext *context; + GLuint pbo; + movit::ResourcePool *resource_pool = nullptr; + QImage grabbed_image; + QTimer grab_timer; + int last_x = -1, last_y = -1; +}; + +#endif // !defined(_ANALYZER_H) diff --git a/nageru/analyzer.ui b/nageru/analyzer.ui new file mode 100644 index 0000000..9cf137e --- /dev/null +++ b/nageru/analyzer.ui @@ -0,0 +1,366 @@ + + + Analyzer + + + + 0 + 0 + 845 + 472 + + + + Analyzer + + + + + + + + + Qt::Vertical + + + + 20 + 5 + + + + + + + + 0 + + + + + Qt::Horizontal + + + + 5 + 20 + + + + + + + + true + + + false + + + background: rgb(233, 185, 110) + + + + + + + Qt::Horizontal + + + + 5 + 20 + + + + + + + + + + + + + + + + + + + Grab every: + + + + + + + + + + Grab + + + + + + + + + false + + + background: rgb(173, 127, 168) + + + + + + + + 0 + 0 + + + + RGB histogram + + + Qt::AlignCenter + + + + + + + Qt::Vertical + + + + 20 + 5 + + + + + + + + + + + + Qt::Vertical + + + + 20 + 5 + + + + + + + + 0 + + + + + Qt::Horizontal + + + + 5 + 20 + + + + + + + + + 1 + 1 + + + + CrossCursor + + + true + + + false + + + background: color(0,0,0) + + + + + + true + + + + + + + Qt::Horizontal + + + + 5 + 20 + + + + + + + + + + Grabbed frame + + + Qt::AlignCenter + + + + + + + Selected coordinate (x,y): (none) + + + Qt::AlignCenter + + + + + + + + + + + + + + + + + + + + + + Color (8-bit sRGB): + + + Qt::AlignCenter + + + + + + + + + Green: + + + + + + + — + + + + + + + — + + + + + + + Hex: + + + + + + + — + + + + + + + Blue: + + + + + + + #— + + + + + + + Red: + + + + + + + + + + + + + Qt::Vertical + + + + 20 + 5 + + + + + + + + + + + + GLWidget + QWidget +
glwidget.h
+ + + QCustomPlot + QWidget +
qcustomplot.h
+ 1 + + + + + diff --git a/nageru/audio_encoder.cpp b/nageru/audio_encoder.cpp new file mode 100644 index 0000000..e33d218 --- /dev/null +++ b/nageru/audio_encoder.cpp @@ -0,0 +1,197 @@ +#include "audio_encoder.h" + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +} + +#include +#include +#include +#include +#include +#include +#include + +#include "defs.h" +#include "mux.h" +#include "timebase.h" + +using namespace std; + +AudioEncoder::AudioEncoder(const string &codec_name, int bit_rate, const AVOutputFormat *oformat) +{ + AVCodec *codec = avcodec_find_encoder_by_name(codec_name.c_str()); + if (codec == nullptr) { + fprintf(stderr, "ERROR: Could not find codec '%s'\n", codec_name.c_str()); + exit(1); + } + + ctx = avcodec_alloc_context3(codec); + ctx->bit_rate = bit_rate; + ctx->sample_rate = OUTPUT_FREQUENCY; + ctx->sample_fmt = codec->sample_fmts[0]; + ctx->channels = 2; + ctx->channel_layout = AV_CH_LAYOUT_STEREO; + ctx->time_base = AVRational{1, TIMEBASE}; + if (oformat->flags & AVFMT_GLOBALHEADER) { + ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + } + if (avcodec_open2(ctx, codec, NULL) < 0) { + fprintf(stderr, "Could not open codec '%s'\n", codec_name.c_str()); + exit(1); + } + + resampler = avresample_alloc_context(); + if (resampler == nullptr) { + fprintf(stderr, "Allocating resampler failed.\n"); + exit(1); + } + + av_opt_set_int(resampler, "in_channel_layout", AV_CH_LAYOUT_STEREO, 0); + av_opt_set_int(resampler, "out_channel_layout", AV_CH_LAYOUT_STEREO, 0); + av_opt_set_int(resampler, "in_sample_rate", OUTPUT_FREQUENCY, 0); + av_opt_set_int(resampler, "out_sample_rate", OUTPUT_FREQUENCY, 0); + av_opt_set_int(resampler, "in_sample_fmt", AV_SAMPLE_FMT_FLT, 0); + av_opt_set_int(resampler, "out_sample_fmt", ctx->sample_fmt, 0); + + if (avresample_open(resampler) < 0) { + fprintf(stderr, "Could not open resample context.\n"); + exit(1); + } + + audio_frame = av_frame_alloc(); +} + +AudioEncoder::~AudioEncoder() +{ + av_frame_free(&audio_frame); + avresample_free(&resampler); + avcodec_free_context(&ctx); +} + +void AudioEncoder::encode_audio(const vector &audio, int64_t audio_pts) +{ + if (ctx->frame_size == 0) { + // No queueing needed. + assert(audio_queue.empty()); + assert(audio.size() % 2 == 0); + encode_audio_one_frame(&audio[0], audio.size() / 2, audio_pts); + return; + } + + int64_t sample_offset = audio_queue.size(); + + audio_queue.insert(audio_queue.end(), audio.begin(), audio.end()); + size_t sample_num; + for (sample_num = 0; + sample_num + ctx->frame_size * 2 <= audio_queue.size(); + sample_num += ctx->frame_size * 2) { + int64_t adjusted_audio_pts = audio_pts + (int64_t(sample_num) - sample_offset) * TIMEBASE / (OUTPUT_FREQUENCY * 2); + encode_audio_one_frame(&audio_queue[sample_num], + ctx->frame_size, + adjusted_audio_pts); + } + audio_queue.erase(audio_queue.begin(), audio_queue.begin() + sample_num); + + last_pts = audio_pts + audio.size() * TIMEBASE / (OUTPUT_FREQUENCY * 2); +} + +void AudioEncoder::encode_audio_one_frame(const float *audio, size_t num_samples, int64_t audio_pts) +{ + audio_frame->pts = audio_pts; + audio_frame->nb_samples = num_samples; + audio_frame->channel_layout = AV_CH_LAYOUT_STEREO; + audio_frame->format = ctx->sample_fmt; + audio_frame->sample_rate = OUTPUT_FREQUENCY; + + if (av_samples_alloc(audio_frame->data, nullptr, 2, num_samples, ctx->sample_fmt, 0) < 0) { + fprintf(stderr, "Could not allocate %ld samples.\n", num_samples); + exit(1); + } + + if (avresample_convert(resampler, audio_frame->data, 0, num_samples, + (uint8_t **)&audio, 0, num_samples) < 0) { + fprintf(stderr, "Audio conversion failed.\n"); + exit(1); + } + + int err = avcodec_send_frame(ctx, audio_frame); + if (err < 0) { + fprintf(stderr, "avcodec_send_frame() failed with error %d\n", err); + exit(1); + } + + for ( ;; ) { // Termination condition within loop. + AVPacket pkt; + av_init_packet(&pkt); + pkt.data = nullptr; + pkt.size = 0; + int err = avcodec_receive_packet(ctx, &pkt); + if (err == 0) { + pkt.stream_index = 1; + pkt.flags = 0; + for (Mux *mux : muxes) { + mux->add_packet(pkt, pkt.pts, pkt.dts); + } + av_packet_unref(&pkt); + } else if (err == AVERROR(EAGAIN)) { + break; + } else { + fprintf(stderr, "avcodec_receive_frame() failed with error %d\n", err); + exit(1); + } + } + + av_freep(&audio_frame->data[0]); + av_frame_unref(audio_frame); +} + +void AudioEncoder::encode_last_audio() +{ + if (!audio_queue.empty()) { + // Last frame can be whatever size we want. + assert(audio_queue.size() % 2 == 0); + encode_audio_one_frame(&audio_queue[0], audio_queue.size() / 2, last_pts); + audio_queue.clear(); + } + + if (ctx->codec->capabilities & AV_CODEC_CAP_DELAY) { + // Collect any delayed frames. + for ( ;; ) { + AVPacket pkt; + av_init_packet(&pkt); + pkt.data = nullptr; + pkt.size = 0; + int err = avcodec_receive_packet(ctx, &pkt); + if (err == 0) { + pkt.stream_index = 1; + pkt.flags = 0; + for (Mux *mux : muxes) { + mux->add_packet(pkt, pkt.pts, pkt.dts); + } + av_packet_unref(&pkt); + } else if (err == AVERROR_EOF) { + break; + } else { + fprintf(stderr, "avcodec_receive_frame() failed with error %d\n", err); + exit(1); + } + } + } +} + +AVCodecParametersWithDeleter AudioEncoder::get_codec_parameters() +{ + AVCodecParameters *codecpar = avcodec_parameters_alloc(); + avcodec_parameters_from_context(codecpar, ctx); + return AVCodecParametersWithDeleter(codecpar); +} diff --git a/nageru/audio_encoder.h b/nageru/audio_encoder.h new file mode 100644 index 0000000..93adbaf --- /dev/null +++ b/nageru/audio_encoder.h @@ -0,0 +1,47 @@ +// A class to encode audio (using ffmpeg) and send it to a Mux. + +#ifndef _AUDIO_ENCODER_H +#define _AUDIO_ENCODER_H 1 + +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +} + +#include "ffmpeg_raii.h" + +class Mux; + +class AudioEncoder { +public: + AudioEncoder(const std::string &codec_name, int bit_rate, const AVOutputFormat *oformat); + ~AudioEncoder(); + + void add_mux(Mux *mux) { // Does not take ownership. + muxes.push_back(mux); + } + void encode_audio(const std::vector &audio, int64_t audio_pts); + void encode_last_audio(); + + AVCodecParametersWithDeleter get_codec_parameters(); + +private: + void encode_audio_one_frame(const float *audio, size_t num_samples, int64_t audio_pts); + + std::vector audio_queue; + int64_t last_pts = 0; // The first pts after all audio we've encoded. + + AVCodecContext *ctx; + AVAudioResampleContext *resampler; + AVFrame *audio_frame = nullptr; + std::vector muxes; +}; + +#endif // !defined(_AUDIO_ENCODER_H) diff --git a/nageru/audio_expanded_view.ui b/nageru/audio_expanded_view.ui new file mode 100644 index 0000000..e71e153 --- /dev/null +++ b/nageru/audio_expanded_view.ui @@ -0,0 +1,564 @@ + + + AudioExpandedView + + + + 0 + 0 + 312 + 484 + + + + AudioExpandedView + + + + + + + DejaVu Sans + 75 + true + true + + + + Bus name + + + Qt::AlignCenter + + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + + 31 + 31 + + + + -100 + + + 100 + + + 100 + + + 50.000000000000000 + + + true + + + + + + + Stereo: 100% + + + + + + + + + Qt::Horizontal + + + + + + + 8 + + + + + Lo-cut + + + + + + + + + + + + 31 + 31 + + + + -150 + + + 150 + + + 60.000000000000000 + + + true + + + + + + + Treble: +0.0 dB + + + + + + + + + + + + 31 + 31 + + + + -150 + + + 150 + + + 60.000000000000000 + + + true + + + + + + + Mid: +0.0 dB + + + + + + + + + + + + 31 + 31 + + + + -150 + + + 150 + + + 60.000000000000000 + + + true + + + + + + + Bass: +0.0 dB + + + + + + + + + Qt::Horizontal + + + + + + + 8 + + + + + Auto gain staging + + + true + + + + + + + + + + + + 31 + 31 + + + + -300 + + + 300 + + + 60.000000000000000 + + + true + + + + + + + Gain: +0.0 dB + + + + + + + + + Qt::Horizontal + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Compressor + + + true + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + + + Threshold + + + Qt::AlignCenter + + + + + + + + + + 64 + 64 + + + + -400 + + + 0 + + + -260 + + + 30.000000000000000 + + + true + + + + + + + + + -10.0 dB + + + Qt::AlignCenter + + + + + + + + + 0 + + + + + Reduction + + + Qt::AlignCenter + + + + + + + + + + 16777215 + 16777215 + + + + + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + + + + + + 20 + 16777215 + + + + + + + + + + + 60 + 0 + + + + -40.0 + + + Qt::AlignCenter + + + + + + + + + + + 0 + + + 0 + + + + + + 0 + 0 + + + + + 40 + 22 + + + + + 8 + + + + QPushButton:checked { background: rgba(255,0,0,80); } + + + Mute + + + true + + + + + + + + + + + 1000 + + + 10 + + + 100 + + + Qt::Vertical + + + + + + + + + + 60 + 0 + + + + +0.0 dB + + + Qt::AlignCenter + + + + + + + + + + + + VUMeter + QWidget +
vumeter.h
+ 1 + + + ClickableLabel + QLabel +
clickable_label.h
+ + + NonLinearFader + QSlider +
nonlinear_fader.h
+ + + EllipsisLabel + QLabel +
ellipsis_label.h
+ + + CompressionReductionMeter + QWidget +
compression_reduction_meter.h
+ 1 + + + + + diff --git a/nageru/audio_miniview.ui b/nageru/audio_miniview.ui new file mode 100644 index 0000000..7aa1aa0 --- /dev/null +++ b/nageru/audio_miniview.ui @@ -0,0 +1,358 @@ + + + AudioMiniView + + + + 0 + 0 + 139 + 300 + + + + + 0 + 0 + + + + + 139 + 0 + + + + + 139 + 16777215 + + + + Form + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + 0 + 1 + + + + true + + + QFrame::Panel + + + QFrame::Plain + + + 0 + + + + 6 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + 0 + + + + + 6 + + + + + + 1 + 0 + + + + + 0 + 0 + + + + Channel description + + + Qt::AlignCenter + + + + + + + 0 + + + + + 0 + + + 4 + + + + + 0 + + + 0 + + + + + + 0 + 1 + + + + + 16 + 0 + + + + + 1 + 0 + + + + + 0 + 0 + + + + + + + + + 255 + 255 + 255 + + + + + + + 5 + 239 + 111 + + + + + + + + + 255 + 255 + 255 + + + + + + + 5 + 239 + 111 + + + + + + + + + 5 + 239 + 111 + + + + + + + 5 + 239 + 111 + + + + + + + + true + + + + + + + + + + 30 + 0 + + + + -0.0 + + + Qt::AlignCenter + + + + + + + + + 4 + + + + + 0 + + + + + 0 + + + 1000 + + + 10 + + + 100 + + + Qt::Vertical + + + QSlider::NoTicks + + + 30 + + + + + + + + + + 0 + 0 + + + + + 30 + 0 + + + + 0 + + + +0.0 dB + + + Qt::AlignCenter + + + + + + + + + + + + + Qt::Vertical + + + + + + + + + + + + + VUMeter + QWidget +
vumeter.h
+ 1 + + + EllipsisLabel + QLabel +
ellipsis_label.h
+ + + NonLinearFader + QSlider +
nonlinear_fader.h
+ + + ClickableLabel + QLabel +
clickable_label.h
+ + + + + diff --git a/nageru/audio_mixer.cpp b/nageru/audio_mixer.cpp new file mode 100644 index 0000000..9e7dd59 --- /dev/null +++ b/nageru/audio_mixer.cpp @@ -0,0 +1,1195 @@ +#include "audio_mixer.h" + +#include +#include +#include +#include +#ifdef __SSE2__ +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db.h" +#include "flags.h" +#include "metrics.h" +#include "state.pb.h" +#include "timebase.h" + +using namespace bmusb; +using namespace std; +using namespace std::chrono; +using namespace std::placeholders; + +namespace { + +// TODO: If these prove to be a bottleneck, they can be SSSE3-optimized +// (usually including multiple channels at a time). + +void convert_fixed16_to_fp32(float *dst, size_t out_channel, size_t out_num_channels, + const uint8_t *src, size_t in_channel, size_t in_num_channels, + size_t num_samples) +{ + assert(in_channel < in_num_channels); + assert(out_channel < out_num_channels); + src += in_channel * 2; + dst += out_channel; + + for (size_t i = 0; i < num_samples; ++i) { + int16_t s = le16toh(*(int16_t *)src); + *dst = s * (1.0f / 32768.0f); + + src += 2 * in_num_channels; + dst += out_num_channels; + } +} + +void convert_fixed24_to_fp32(float *dst, size_t out_channel, size_t out_num_channels, + const uint8_t *src, size_t in_channel, size_t in_num_channels, + size_t num_samples) +{ + assert(in_channel < in_num_channels); + assert(out_channel < out_num_channels); + src += in_channel * 3; + dst += out_channel; + + for (size_t i = 0; i < num_samples; ++i) { + uint32_t s1 = src[0]; + uint32_t s2 = src[1]; + uint32_t s3 = src[2]; + uint32_t s = s1 | (s1 << 8) | (s2 << 16) | (s3 << 24); + *dst = int(s) * (1.0f / 2147483648.0f); + + src += 3 * in_num_channels; + dst += out_num_channels; + } +} + +void convert_fixed32_to_fp32(float *dst, size_t out_channel, size_t out_num_channels, + const uint8_t *src, size_t in_channel, size_t in_num_channels, + size_t num_samples) +{ + assert(in_channel < in_num_channels); + assert(out_channel < out_num_channels); + src += in_channel * 4; + dst += out_channel; + + for (size_t i = 0; i < num_samples; ++i) { + int32_t s = le32toh(*(int32_t *)src); + *dst = s * (1.0f / 2147483648.0f); + + src += 4 * in_num_channels; + dst += out_num_channels; + } +} + +float find_peak_plain(const float *samples, size_t num_samples) __attribute__((unused)); + +float find_peak_plain(const float *samples, size_t num_samples) +{ + float m = fabs(samples[0]); + for (size_t i = 1; i < num_samples; ++i) { + m = max(m, fabs(samples[i])); + } + return m; +} + +#ifdef __SSE__ +static inline float horizontal_max(__m128 m) +{ + __m128 tmp = _mm_shuffle_ps(m, m, _MM_SHUFFLE(1, 0, 3, 2)); + m = _mm_max_ps(m, tmp); + tmp = _mm_shuffle_ps(m, m, _MM_SHUFFLE(2, 3, 0, 1)); + m = _mm_max_ps(m, tmp); + return _mm_cvtss_f32(m); +} + +float find_peak(const float *samples, size_t num_samples) +{ + const __m128 abs_mask = _mm_castsi128_ps(_mm_set1_epi32(0x7fffffffu)); + __m128 m = _mm_setzero_ps(); + for (size_t i = 0; i < (num_samples & ~3); i += 4) { + __m128 x = _mm_loadu_ps(samples + i); + x = _mm_and_ps(x, abs_mask); + m = _mm_max_ps(m, x); + } + float result = horizontal_max(m); + + for (size_t i = (num_samples & ~3); i < num_samples; ++i) { + result = max(result, fabs(samples[i])); + } + +#if 0 + // Self-test. We should be bit-exact the same. + float reference_result = find_peak_plain(samples, num_samples); + if (result != reference_result) { + fprintf(stderr, "Error: Peak is %f [%f %f %f %f]; should be %f.\n", + result, + _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(0, 0, 0, 0))), + _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(1, 1, 1, 1))), + _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(2, 2, 2, 2))), + _mm_cvtss_f32(_mm_shuffle_ps(m, m, _MM_SHUFFLE(3, 3, 3, 3))), + reference_result); + abort(); + } +#endif + return result; +} +#else +float find_peak(const float *samples, size_t num_samples) +{ + return find_peak_plain(samples, num_samples); +} +#endif + +void deinterleave_samples(const vector &in, vector *out_l, vector *out_r) +{ + size_t num_samples = in.size() / 2; + out_l->resize(num_samples); + out_r->resize(num_samples); + + const float *inptr = in.data(); + float *lptr = &(*out_l)[0]; + float *rptr = &(*out_r)[0]; + for (size_t i = 0; i < num_samples; ++i) { + *lptr++ = *inptr++; + *rptr++ = *inptr++; + } +} + +} // namespace + +AudioMixer::AudioMixer(unsigned num_capture_cards, unsigned num_ffmpeg_inputs) + : num_capture_cards(num_capture_cards), + num_ffmpeg_inputs(num_ffmpeg_inputs), + ffmpeg_inputs(new AudioDevice[num_ffmpeg_inputs]), + limiter(OUTPUT_FREQUENCY), + correlation(OUTPUT_FREQUENCY) +{ + for (unsigned bus_index = 0; bus_index < MAX_BUSES; ++bus_index) { + locut[bus_index].init(FILTER_HPF, 2); + eq[bus_index][EQ_BAND_BASS].init(FILTER_LOW_SHELF, 1); + // Note: EQ_BAND_MID isn't used (see comments in apply_eq()). + eq[bus_index][EQ_BAND_TREBLE].init(FILTER_HIGH_SHELF, 1); + compressor[bus_index].reset(new StereoCompressor(OUTPUT_FREQUENCY)); + level_compressor[bus_index].reset(new StereoCompressor(OUTPUT_FREQUENCY)); + + set_bus_settings(bus_index, get_default_bus_settings()); + } + set_limiter_enabled(global_flags.limiter_enabled); + set_final_makeup_gain_auto(global_flags.final_makeup_gain_auto); + + r128.init(2, OUTPUT_FREQUENCY); + r128.integr_start(); + + // hlen=16 is pretty low quality, but we use quite a bit of CPU otherwise, + // and there's a limit to how important the peak meter is. + peak_resampler.setup(OUTPUT_FREQUENCY, OUTPUT_FREQUENCY * 4, /*num_channels=*/2, /*hlen=*/16, /*frel=*/1.0); + + global_audio_mixer = this; + alsa_pool.init(); + + if (!global_flags.input_mapping_filename.empty()) { + // Must happen after ALSAPool is initialized, as it needs to know the card list. + current_mapping_mode = MappingMode::MULTICHANNEL; + InputMapping new_input_mapping; + if (!load_input_mapping_from_file(get_devices(), + global_flags.input_mapping_filename, + &new_input_mapping)) { + fprintf(stderr, "Failed to load input mapping from '%s', exiting.\n", + global_flags.input_mapping_filename.c_str()); + exit(1); + } + set_input_mapping(new_input_mapping); + } else { + set_simple_input(/*card_index=*/0); + if (global_flags.multichannel_mapping_mode) { + current_mapping_mode = MappingMode::MULTICHANNEL; + } + } + + global_metrics.add("audio_loudness_short_lufs", &metric_audio_loudness_short_lufs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_loudness_integrated_lufs", &metric_audio_loudness_integrated_lufs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_loudness_range_low_lufs", &metric_audio_loudness_range_low_lufs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_loudness_range_high_lufs", &metric_audio_loudness_range_high_lufs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_peak_dbfs", &metric_audio_peak_dbfs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_final_makeup_gain_db", &metric_audio_final_makeup_gain_db, Metrics::TYPE_GAUGE); + global_metrics.add("audio_correlation", &metric_audio_correlation, Metrics::TYPE_GAUGE); +} + +void AudioMixer::reset_resampler(DeviceSpec device_spec) +{ + lock_guard lock(audio_mutex); + reset_resampler_mutex_held(device_spec); +} + +void AudioMixer::reset_resampler_mutex_held(DeviceSpec device_spec) +{ + AudioDevice *device = find_audio_device(device_spec); + + if (device->interesting_channels.empty()) { + device->resampling_queue.reset(); + } else { + device->resampling_queue.reset(new ResamplingQueue( + device_spec, device->capture_frequency, OUTPUT_FREQUENCY, device->interesting_channels.size(), + global_flags.audio_queue_length_ms * 0.001)); + } +} + +bool AudioMixer::add_audio(DeviceSpec device_spec, const uint8_t *data, unsigned num_samples, AudioFormat audio_format, int64_t frame_length, steady_clock::time_point frame_time) +{ + AudioDevice *device = find_audio_device(device_spec); + + unique_lock lock(audio_mutex, defer_lock); + if (!lock.try_lock_for(chrono::milliseconds(10))) { + return false; + } + if (device->resampling_queue == nullptr) { + // No buses use this device; throw it away. + return true; + } + + unsigned num_channels = device->interesting_channels.size(); + assert(num_channels > 0); + + // Convert the audio to fp32. + unique_ptr audio(new float[num_samples * num_channels]); + unsigned channel_index = 0; + for (auto channel_it = device->interesting_channels.cbegin(); channel_it != device->interesting_channels.end(); ++channel_it, ++channel_index) { + switch (audio_format.bits_per_sample) { + case 0: + assert(num_samples == 0); + break; + case 16: + convert_fixed16_to_fp32(audio.get(), channel_index, num_channels, data, *channel_it, audio_format.num_channels, num_samples); + break; + case 24: + convert_fixed24_to_fp32(audio.get(), channel_index, num_channels, data, *channel_it, audio_format.num_channels, num_samples); + break; + case 32: + convert_fixed32_to_fp32(audio.get(), channel_index, num_channels, data, *channel_it, audio_format.num_channels, num_samples); + break; + default: + fprintf(stderr, "Cannot handle audio with %u bits per sample\n", audio_format.bits_per_sample); + assert(false); + } + } + + // If we changed frequency since last frame, we'll need to reset the resampler. + if (audio_format.sample_rate != device->capture_frequency) { + device->capture_frequency = audio_format.sample_rate; + reset_resampler_mutex_held(device_spec); + } + + // Now add it. + device->resampling_queue->add_input_samples(frame_time, audio.get(), num_samples, ResamplingQueue::ADJUST_RATE); + return true; +} + +bool AudioMixer::add_silence(DeviceSpec device_spec, unsigned samples_per_frame, unsigned num_frames, int64_t frame_length) +{ + AudioDevice *device = find_audio_device(device_spec); + + unique_lock lock(audio_mutex, defer_lock); + if (!lock.try_lock_for(chrono::milliseconds(10))) { + return false; + } + if (device->resampling_queue == nullptr) { + // No buses use this device; throw it away. + return true; + } + + unsigned num_channels = device->interesting_channels.size(); + assert(num_channels > 0); + + vector silence(samples_per_frame * num_channels, 0.0f); + for (unsigned i = 0; i < num_frames; ++i) { + device->resampling_queue->add_input_samples(steady_clock::now(), silence.data(), samples_per_frame, ResamplingQueue::DO_NOT_ADJUST_RATE); + } + return true; +} + +bool AudioMixer::silence_card(DeviceSpec device_spec, bool silence) +{ + AudioDevice *device = find_audio_device(device_spec); + + unique_lock lock(audio_mutex, defer_lock); + if (!lock.try_lock_for(chrono::milliseconds(10))) { + return false; + } + + if (device->silenced && !silence) { + reset_resampler_mutex_held(device_spec); + } + device->silenced = silence; + return true; +} + +AudioMixer::BusSettings AudioMixer::get_default_bus_settings() +{ + BusSettings settings; + settings.fader_volume_db = 0.0f; + settings.muted = false; + settings.locut_enabled = global_flags.locut_enabled; + settings.stereo_width = 1.0f; + for (unsigned band_index = 0; band_index < NUM_EQ_BANDS; ++band_index) { + settings.eq_level_db[band_index] = 0.0f; + } + settings.gain_staging_db = global_flags.initial_gain_staging_db; + settings.level_compressor_enabled = global_flags.gain_staging_auto; + settings.compressor_threshold_dbfs = ref_level_dbfs - 12.0f; // -12 dB. + settings.compressor_enabled = global_flags.compressor_enabled; + return settings; +} + +AudioMixer::BusSettings AudioMixer::get_bus_settings(unsigned bus_index) const +{ + lock_guard lock(audio_mutex); + BusSettings settings; + settings.fader_volume_db = fader_volume_db[bus_index]; + settings.muted = mute[bus_index]; + settings.locut_enabled = locut_enabled[bus_index]; + settings.stereo_width = stereo_width[bus_index]; + for (unsigned band_index = 0; band_index < NUM_EQ_BANDS; ++band_index) { + settings.eq_level_db[band_index] = eq_level_db[bus_index][band_index]; + } + settings.gain_staging_db = gain_staging_db[bus_index]; + settings.level_compressor_enabled = level_compressor_enabled[bus_index]; + settings.compressor_threshold_dbfs = compressor_threshold_dbfs[bus_index]; + settings.compressor_enabled = compressor_enabled[bus_index]; + return settings; +} + +void AudioMixer::set_bus_settings(unsigned bus_index, const AudioMixer::BusSettings &settings) +{ + lock_guard lock(audio_mutex); + fader_volume_db[bus_index] = settings.fader_volume_db; + mute[bus_index] = settings.muted; + locut_enabled[bus_index] = settings.locut_enabled; + stereo_width[bus_index] = settings.stereo_width; + for (unsigned band_index = 0; band_index < NUM_EQ_BANDS; ++band_index) { + eq_level_db[bus_index][band_index] = settings.eq_level_db[band_index]; + } + gain_staging_db[bus_index] = settings.gain_staging_db; + last_gain_staging_db[bus_index] = gain_staging_db[bus_index]; + level_compressor_enabled[bus_index] = settings.level_compressor_enabled; + compressor_threshold_dbfs[bus_index] = settings.compressor_threshold_dbfs; + compressor_enabled[bus_index] = settings.compressor_enabled; +} + +AudioMixer::AudioDevice *AudioMixer::find_audio_device(DeviceSpec device) +{ + switch (device.type) { + case InputSourceType::CAPTURE_CARD: + return &video_cards[device.index]; + case InputSourceType::ALSA_INPUT: + return &alsa_inputs[device.index]; + case InputSourceType::FFMPEG_VIDEO_INPUT: + return &ffmpeg_inputs[device.index]; + case InputSourceType::SILENCE: + default: + assert(false); + } + return nullptr; +} + +// Get a pointer to the given channel from the given device. +// The channel must be picked out earlier and resampled. +void AudioMixer::find_sample_src_from_device(const map> &samples_card, DeviceSpec device_spec, int source_channel, const float **srcptr, unsigned *stride) +{ + static float zero = 0.0f; + if (source_channel == -1 || device_spec.type == InputSourceType::SILENCE) { + *srcptr = &zero; + *stride = 0; + return; + } + AudioDevice *device = find_audio_device(device_spec); + assert(device->interesting_channels.count(source_channel) != 0); + unsigned channel_index = 0; + for (int channel : device->interesting_channels) { + if (channel == source_channel) break; + ++channel_index; + } + assert(channel_index < device->interesting_channels.size()); + const auto it = samples_card.find(device_spec); + assert(it != samples_card.end()); + *srcptr = &(it->second)[channel_index]; + *stride = device->interesting_channels.size(); +} + +// TODO: Can be SSSE3-optimized if need be. +void AudioMixer::fill_audio_bus(const map> &samples_card, const InputMapping::Bus &bus, unsigned num_samples, float stereo_width, float *output) +{ + if (bus.device.type == InputSourceType::SILENCE) { + memset(output, 0, num_samples * 2 * sizeof(*output)); + } else { + assert(bus.device.type == InputSourceType::CAPTURE_CARD || + bus.device.type == InputSourceType::ALSA_INPUT || + bus.device.type == InputSourceType::FFMPEG_VIDEO_INPUT); + const float *lsrc, *rsrc; + unsigned lstride, rstride; + float *dptr = output; + find_sample_src_from_device(samples_card, bus.device, bus.source_channel[0], &lsrc, &lstride); + find_sample_src_from_device(samples_card, bus.device, bus.source_channel[1], &rsrc, &rstride); + + // Apply stereo width settings. Set stereo width w to a 0..1 range instead of + // -1..1, since it makes for much easier calculations (so 0.5 = completely mono). + // Then, what we want is + // + // L' = wL + (1-w)R = R + w(L-R) + // R' = wR + (1-w)L = L + w(R-L) + // + // This can be further simplified calculation-wise by defining the weighted + // difference signal D = w(R-L), so that: + // + // L' = R - D + // R' = L + D + float w = 0.5f * stereo_width + 0.5f; + if (bus.source_channel[0] == bus.source_channel[1]) { + // Mono anyway, so no need to bother. + w = 1.0f; + } else if (fabs(w) < 1e-3) { + // Perfect inverse. + swap(lsrc, rsrc); + swap(lstride, rstride); + w = 1.0f; + } + if (fabs(w - 1.0f) < 1e-3) { + // No calculations needed for stereo_width = 1. + for (unsigned i = 0; i < num_samples; ++i) { + *dptr++ = *lsrc; + *dptr++ = *rsrc; + lsrc += lstride; + rsrc += rstride; + } + } else { + // General case. + for (unsigned i = 0; i < num_samples; ++i) { + float left = *lsrc, right = *rsrc; + float diff = w * (right - left); + *dptr++ = right - diff; + *dptr++ = left + diff; + lsrc += lstride; + rsrc += rstride; + } + } + } +} + +vector AudioMixer::get_active_devices() const +{ + vector ret; + for (unsigned card_index = 0; card_index < MAX_VIDEO_CARDS; ++card_index) { + const DeviceSpec device_spec{InputSourceType::CAPTURE_CARD, card_index}; + if (!find_audio_device(device_spec)->interesting_channels.empty()) { + ret.push_back(device_spec); + } + } + for (unsigned card_index = 0; card_index < MAX_ALSA_CARDS; ++card_index) { + const DeviceSpec device_spec{InputSourceType::ALSA_INPUT, card_index}; + if (!find_audio_device(device_spec)->interesting_channels.empty()) { + ret.push_back(device_spec); + } + } + for (unsigned card_index = 0; card_index < num_ffmpeg_inputs; ++card_index) { + const DeviceSpec device_spec{InputSourceType::FFMPEG_VIDEO_INPUT, card_index}; + if (!find_audio_device(device_spec)->interesting_channels.empty()) { + ret.push_back(device_spec); + } + } + return ret; +} + +namespace { + +void apply_gain(float db, float last_db, vector *samples) +{ + if (fabs(db - last_db) < 1e-3) { + // Constant over this frame. + const float gain = from_db(db); + for (size_t i = 0; i < samples->size(); ++i) { + (*samples)[i] *= gain; + } + } else { + // We need to do a fade. + unsigned num_samples = samples->size() / 2; + float gain = from_db(last_db); + const float gain_inc = pow(from_db(db - last_db), 1.0 / num_samples); + for (size_t i = 0; i < num_samples; ++i) { + (*samples)[i * 2 + 0] *= gain; + (*samples)[i * 2 + 1] *= gain; + gain *= gain_inc; + } + } +} + +} // namespace + +vector AudioMixer::get_output(steady_clock::time_point ts, unsigned num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy) +{ + map> samples_card; + vector samples_bus; + + lock_guard lock(audio_mutex); + + // Pick out all the interesting channels from all the cards. + for (const DeviceSpec &device_spec : get_active_devices()) { + AudioDevice *device = find_audio_device(device_spec); + samples_card[device_spec].resize(num_samples * device->interesting_channels.size()); + if (device->silenced) { + memset(&samples_card[device_spec][0], 0, samples_card[device_spec].size() * sizeof(float)); + } else { + device->resampling_queue->get_output_samples( + ts, + &samples_card[device_spec][0], + num_samples, + rate_adjustment_policy); + } + } + + vector samples_out, left, right; + samples_out.resize(num_samples * 2); + samples_bus.resize(num_samples * 2); + for (unsigned bus_index = 0; bus_index < input_mapping.buses.size(); ++bus_index) { + fill_audio_bus(samples_card, input_mapping.buses[bus_index], num_samples, stereo_width[bus_index], &samples_bus[0]); + apply_eq(bus_index, &samples_bus); + + { + lock_guard lock(compressor_mutex); + + // Apply a level compressor to get the general level right. + // Basically, if it's over about -40 dBFS, we squeeze it down to that level + // (or more precisely, near it, since we don't use infinite ratio), + // then apply a makeup gain to get it to -14 dBFS. -14 dBFS is, of course, + // entirely arbitrary, but from practical tests with speech, it seems to + // put ut around -23 LUFS, so it's a reasonable starting point for later use. + if (level_compressor_enabled[bus_index]) { + float threshold = 0.01f; // -40 dBFS. + float ratio = 20.0f; + float attack_time = 0.5f; + float release_time = 20.0f; + float makeup_gain = from_db(ref_level_dbfs - (-40.0f)); // +26 dB. + level_compressor[bus_index]->process(samples_bus.data(), samples_bus.size() / 2, threshold, ratio, attack_time, release_time, makeup_gain); + gain_staging_db[bus_index] = to_db(level_compressor[bus_index]->get_attenuation() * makeup_gain); + } else { + // Just apply the gain we already had. + float db = gain_staging_db[bus_index]; + float last_db = last_gain_staging_db[bus_index]; + apply_gain(db, last_db, &samples_bus); + } + last_gain_staging_db[bus_index] = gain_staging_db[bus_index]; + +#if 0 + printf("level=%f (%+5.2f dBFS) attenuation=%f (%+5.2f dB) end_result=%+5.2f dB\n", + level_compressor.get_level(), to_db(level_compressor.get_level()), + level_compressor.get_attenuation(), to_db(level_compressor.get_attenuation()), + to_db(level_compressor.get_level() * level_compressor.get_attenuation() * makeup_gain)); +#endif + + // The real compressor. + if (compressor_enabled[bus_index]) { + float threshold = from_db(compressor_threshold_dbfs[bus_index]); + float ratio = 20.0f; + float attack_time = 0.005f; + float release_time = 0.040f; + float makeup_gain = 2.0f; // +6 dB. + compressor[bus_index]->process(samples_bus.data(), samples_bus.size() / 2, threshold, ratio, attack_time, release_time, makeup_gain); + // compressor_att = compressor.get_attenuation(); + } + } + + add_bus_to_master(bus_index, samples_bus, &samples_out); + deinterleave_samples(samples_bus, &left, &right); + measure_bus_levels(bus_index, left, right); + } + + { + lock_guard lock(compressor_mutex); + + // Finally a limiter at -4 dB (so, -10 dBFS) to take out the worst peaks only. + // Note that since ratio is not infinite, we could go slightly higher than this. + if (limiter_enabled) { + float threshold = from_db(limiter_threshold_dbfs); + float ratio = 30.0f; + float attack_time = 0.0f; // Instant. + float release_time = 0.020f; + float makeup_gain = 1.0f; // 0 dB. + limiter.process(samples_out.data(), samples_out.size() / 2, threshold, ratio, attack_time, release_time, makeup_gain); + // limiter_att = limiter.get_attenuation(); + } + + // printf("limiter=%+5.1f compressor=%+5.1f\n", to_db(limiter_att), to_db(compressor_att)); + } + + // At this point, we are most likely close to +0 LU (at least if the + // faders sum to 0 dB and the compressors are on), but all of our + // measurements have been on raw sample values, not R128 values. + // So we have a final makeup gain to get us to +0 LU; the gain + // adjustments required should be relatively small, and also, the + // offset shouldn't change much (only if the type of audio changes + // significantly). Thus, we shoot for updating this value basically + // “whenever we process buffers”, since the R128 calculation isn't exactly + // something we get out per-sample. + // + // Note that there's a feedback loop here, so we choose a very slow filter + // (half-time of 30 seconds). + double target_loudness_factor, alpha; + double loudness_lu = r128.loudness_M() - ref_level_lufs; + target_loudness_factor = final_makeup_gain * from_db(-loudness_lu); + + // If we're outside +/- 5 LU (after correction), we don't count it as + // a normal signal (probably silence) and don't change the + // correction factor; just apply what we already have. + if (fabs(loudness_lu) >= 5.0 || !final_makeup_gain_auto) { + alpha = 0.0; + } else { + // Formula adapted from + // https://en.wikipedia.org/wiki/Low-pass_filter#Simple_infinite_impulse_response_filter. + const double half_time_s = 30.0; + const double fc_mul_2pi_delta_t = 1.0 / (half_time_s * OUTPUT_FREQUENCY); + alpha = fc_mul_2pi_delta_t / (fc_mul_2pi_delta_t + 1.0); + } + + { + lock_guard lock(compressor_mutex); + double m = final_makeup_gain; + for (size_t i = 0; i < samples_out.size(); i += 2) { + samples_out[i + 0] *= m; + samples_out[i + 1] *= m; + m += (target_loudness_factor - m) * alpha; + } + final_makeup_gain = m; + } + + update_meters(samples_out); + + return samples_out; +} + +namespace { + +void apply_filter_fade(StereoFilter *filter, float *data, unsigned num_samples, float cutoff_hz, float db, float last_db) +{ + // A granularity of 32 samples is an okay tradeoff between speed and + // smoothness; recalculating the filters is pretty expensive, so it's + // good that we don't do this all the time. + static constexpr unsigned filter_granularity_samples = 32; + + const float cutoff_linear = cutoff_hz * 2.0 * M_PI / OUTPUT_FREQUENCY; + if (fabs(db - last_db) < 1e-3) { + // Constant over this frame. + if (fabs(db) > 0.01f) { + filter->render(data, num_samples, cutoff_linear, 0.5f, db / 40.0f); + } + } else { + // We need to do a fade. (Rounding up avoids division by zero.) + unsigned num_blocks = (num_samples + filter_granularity_samples - 1) / filter_granularity_samples; + const float inc_db_norm = (db - last_db) / 40.0f / num_blocks; + float db_norm = db / 40.0f; + for (size_t i = 0; i < num_samples; i += filter_granularity_samples) { + size_t samples_this_block = std::min(num_samples - i, filter_granularity_samples); + filter->render(data + i * 2, samples_this_block, cutoff_linear, 0.5f, db_norm); + db_norm += inc_db_norm; + } + } +} + +} // namespace + +void AudioMixer::apply_eq(unsigned bus_index, vector *samples_bus) +{ + constexpr float bass_freq_hz = 200.0f; + constexpr float treble_freq_hz = 4700.0f; + + // Cut away everything under 120 Hz (or whatever the cutoff is); + // we don't need it for voice, and it will reduce headroom + // and confuse the compressor. (In particular, any hums at 50 or 60 Hz + // should be dampened.) + if (locut_enabled[bus_index]) { + locut[bus_index].render(samples_bus->data(), samples_bus->size() / 2, locut_cutoff_hz * 2.0 * M_PI / OUTPUT_FREQUENCY, 0.5f); + } + + // Apply the rest of the EQ. Since we only have a simple three-band EQ, + // we can implement it with two shelf filters. We use a simple gain to + // set the mid-level filter, and then offset the low and high bands + // from that if we need to. (We could perhaps have folded the gain into + // the next part, but it's so cheap that the trouble isn't worth it.) + // + // If any part of the EQ has changed appreciably since last frame, + // we fade smoothly during the course of this frame. + const float bass_db = eq_level_db[bus_index][EQ_BAND_BASS]; + const float mid_db = eq_level_db[bus_index][EQ_BAND_MID]; + const float treble_db = eq_level_db[bus_index][EQ_BAND_TREBLE]; + + const float last_bass_db = last_eq_level_db[bus_index][EQ_BAND_BASS]; + const float last_mid_db = last_eq_level_db[bus_index][EQ_BAND_MID]; + const float last_treble_db = last_eq_level_db[bus_index][EQ_BAND_TREBLE]; + + assert(samples_bus->size() % 2 == 0); + const unsigned num_samples = samples_bus->size() / 2; + + apply_gain(mid_db, last_mid_db, samples_bus); + + apply_filter_fade(&eq[bus_index][EQ_BAND_BASS], samples_bus->data(), num_samples, bass_freq_hz, bass_db - mid_db, last_bass_db - last_mid_db); + apply_filter_fade(&eq[bus_index][EQ_BAND_TREBLE], samples_bus->data(), num_samples, treble_freq_hz, treble_db - mid_db, last_treble_db - last_mid_db); + + last_eq_level_db[bus_index][EQ_BAND_BASS] = bass_db; + last_eq_level_db[bus_index][EQ_BAND_MID] = mid_db; + last_eq_level_db[bus_index][EQ_BAND_TREBLE] = treble_db; +} + +void AudioMixer::add_bus_to_master(unsigned bus_index, const vector &samples_bus, vector *samples_out) +{ + assert(samples_bus.size() == samples_out->size()); + assert(samples_bus.size() % 2 == 0); + unsigned num_samples = samples_bus.size() / 2; + const float new_volume_db = mute[bus_index] ? -90.0f : fader_volume_db[bus_index].load(); + if (fabs(new_volume_db - last_fader_volume_db[bus_index]) > 1e-3) { + // The volume has changed; do a fade over the course of this frame. + // (We might have some numerical issues here, but it seems to sound OK.) + // For the purpose of fading here, the silence floor is set to -90 dB + // (the fader only goes to -84). + float old_volume = from_db(max(last_fader_volume_db[bus_index], -90.0f)); + float volume = from_db(max(new_volume_db, -90.0f)); + + float volume_inc = pow(volume / old_volume, 1.0 / num_samples); + volume = old_volume; + if (bus_index == 0) { + for (unsigned i = 0; i < num_samples; ++i) { + (*samples_out)[i * 2 + 0] = samples_bus[i * 2 + 0] * volume; + (*samples_out)[i * 2 + 1] = samples_bus[i * 2 + 1] * volume; + volume *= volume_inc; + } + } else { + for (unsigned i = 0; i < num_samples; ++i) { + (*samples_out)[i * 2 + 0] += samples_bus[i * 2 + 0] * volume; + (*samples_out)[i * 2 + 1] += samples_bus[i * 2 + 1] * volume; + volume *= volume_inc; + } + } + } else if (new_volume_db > -90.0f) { + float volume = from_db(new_volume_db); + if (bus_index == 0) { + for (unsigned i = 0; i < num_samples; ++i) { + (*samples_out)[i * 2 + 0] = samples_bus[i * 2 + 0] * volume; + (*samples_out)[i * 2 + 1] = samples_bus[i * 2 + 1] * volume; + } + } else { + for (unsigned i = 0; i < num_samples; ++i) { + (*samples_out)[i * 2 + 0] += samples_bus[i * 2 + 0] * volume; + (*samples_out)[i * 2 + 1] += samples_bus[i * 2 + 1] * volume; + } + } + } + + last_fader_volume_db[bus_index] = new_volume_db; +} + +void AudioMixer::measure_bus_levels(unsigned bus_index, const vector &left, const vector &right) +{ + assert(left.size() == right.size()); + const float volume = mute[bus_index] ? 0.0f : from_db(fader_volume_db[bus_index]); + const float peak_levels[2] = { + find_peak(left.data(), left.size()) * volume, + find_peak(right.data(), right.size()) * volume + }; + for (unsigned channel = 0; channel < 2; ++channel) { + // Compute the current value, including hold and falloff. + // The constants are borrowed from zita-mu1 by Fons Adriaensen. + static constexpr float hold_sec = 0.5f; + static constexpr float falloff_db_sec = 15.0f; // dB/sec falloff after hold. + float current_peak; + PeakHistory &history = peak_history[bus_index][channel]; + history.historic_peak = max(history.historic_peak, peak_levels[channel]); + if (history.age_seconds < hold_sec) { + current_peak = history.last_peak; + } else { + current_peak = history.last_peak * from_db(-falloff_db_sec * (history.age_seconds - hold_sec)); + } + + // See if we have a new peak to replace the old (possibly falling) one. + if (peak_levels[channel] > current_peak) { + history.last_peak = peak_levels[channel]; + history.age_seconds = 0.0f; // Not 100% correct, but more than good enough given our frame sizes. + current_peak = peak_levels[channel]; + } else { + history.age_seconds += float(left.size()) / OUTPUT_FREQUENCY; + } + history.current_level = peak_levels[channel]; + history.current_peak = current_peak; + } +} + +void AudioMixer::update_meters(const vector &samples) +{ + // Upsample 4x to find interpolated peak. + peak_resampler.inp_data = const_cast(samples.data()); + peak_resampler.inp_count = samples.size() / 2; + + vector interpolated_samples; + interpolated_samples.resize(samples.size()); + { + lock_guard lock(audio_measure_mutex); + + while (peak_resampler.inp_count > 0) { // About four iterations. + peak_resampler.out_data = &interpolated_samples[0]; + peak_resampler.out_count = interpolated_samples.size() / 2; + peak_resampler.process(); + size_t out_stereo_samples = interpolated_samples.size() / 2 - peak_resampler.out_count; + peak = max(peak, find_peak(interpolated_samples.data(), out_stereo_samples * 2)); + peak_resampler.out_data = nullptr; + } + } + + // Find R128 levels and L/R correlation. + vector left, right; + deinterleave_samples(samples, &left, &right); + float *ptrs[] = { left.data(), right.data() }; + { + lock_guard lock(audio_measure_mutex); + r128.process(left.size(), ptrs); + correlation.process_samples(samples); + } + + send_audio_level_callback(); +} + +void AudioMixer::reset_meters() +{ + lock_guard lock(audio_measure_mutex); + peak_resampler.reset(); + peak = 0.0f; + r128.reset(); + r128.integr_start(); + correlation.reset(); +} + +void AudioMixer::send_audio_level_callback() +{ + if (audio_level_callback == nullptr) { + return; + } + + lock_guard lock(audio_measure_mutex); + double loudness_s = r128.loudness_S(); + double loudness_i = r128.integrated(); + double loudness_range_low = r128.range_min(); + double loudness_range_high = r128.range_max(); + + metric_audio_loudness_short_lufs = loudness_s; + metric_audio_loudness_integrated_lufs = loudness_i; + metric_audio_loudness_range_low_lufs = loudness_range_low; + metric_audio_loudness_range_high_lufs = loudness_range_high; + metric_audio_peak_dbfs = to_db(peak); + metric_audio_final_makeup_gain_db = to_db(final_makeup_gain); + metric_audio_correlation = correlation.get_correlation(); + + vector bus_levels; + bus_levels.resize(input_mapping.buses.size()); + { + lock_guard lock(compressor_mutex); + for (unsigned bus_index = 0; bus_index < bus_levels.size(); ++bus_index) { + BusLevel &levels = bus_levels[bus_index]; + BusMetrics &metrics = bus_metrics[bus_index]; + + levels.current_level_dbfs[0] = metrics.current_level_dbfs[0] = to_db(peak_history[bus_index][0].current_level); + levels.current_level_dbfs[1] = metrics.current_level_dbfs[1] = to_db(peak_history[bus_index][1].current_level); + levels.peak_level_dbfs[0] = metrics.peak_level_dbfs[0] = to_db(peak_history[bus_index][0].current_peak); + levels.peak_level_dbfs[1] = metrics.peak_level_dbfs[1] = to_db(peak_history[bus_index][1].current_peak); + levels.historic_peak_dbfs = metrics.historic_peak_dbfs = to_db( + max(peak_history[bus_index][0].historic_peak, + peak_history[bus_index][1].historic_peak)); + levels.gain_staging_db = metrics.gain_staging_db = gain_staging_db[bus_index]; + if (compressor_enabled[bus_index]) { + levels.compressor_attenuation_db = metrics.compressor_attenuation_db = -to_db(compressor[bus_index]->get_attenuation()); + } else { + levels.compressor_attenuation_db = 0.0; + metrics.compressor_attenuation_db = 0.0 / 0.0; + } + } + } + + audio_level_callback(loudness_s, to_db(peak), bus_levels, + loudness_i, loudness_range_low, loudness_range_high, + to_db(final_makeup_gain), + correlation.get_correlation()); +} + +map AudioMixer::get_devices() +{ + lock_guard lock(audio_mutex); + + map devices; + for (unsigned card_index = 0; card_index < num_capture_cards; ++card_index) { + const DeviceSpec spec{ InputSourceType::CAPTURE_CARD, card_index }; + const AudioDevice *device = &video_cards[card_index]; + DeviceInfo info; + info.display_name = device->display_name; + info.num_channels = 8; + devices.insert(make_pair(spec, info)); + } + vector available_alsa_devices = alsa_pool.get_devices(); + for (unsigned card_index = 0; card_index < available_alsa_devices.size(); ++card_index) { + const DeviceSpec spec{ InputSourceType::ALSA_INPUT, card_index }; + const ALSAPool::Device &device = available_alsa_devices[card_index]; + DeviceInfo info; + info.display_name = device.display_name(); + info.num_channels = device.num_channels; + info.alsa_name = device.name; + info.alsa_info = device.info; + info.alsa_address = device.address; + devices.insert(make_pair(spec, info)); + } + for (unsigned card_index = 0; card_index < num_ffmpeg_inputs; ++card_index) { + const DeviceSpec spec{ InputSourceType::FFMPEG_VIDEO_INPUT, card_index }; + const AudioDevice *device = &ffmpeg_inputs[card_index]; + DeviceInfo info; + info.display_name = device->display_name; + info.num_channels = 2; + devices.insert(make_pair(spec, info)); + } + return devices; +} + +void AudioMixer::set_display_name(DeviceSpec device_spec, const string &name) +{ + AudioDevice *device = find_audio_device(device_spec); + + lock_guard lock(audio_mutex); + device->display_name = name; +} + +void AudioMixer::serialize_device(DeviceSpec device_spec, DeviceSpecProto *device_spec_proto) +{ + lock_guard lock(audio_mutex); + switch (device_spec.type) { + case InputSourceType::SILENCE: + device_spec_proto->set_type(DeviceSpecProto::SILENCE); + break; + case InputSourceType::CAPTURE_CARD: + device_spec_proto->set_type(DeviceSpecProto::CAPTURE_CARD); + device_spec_proto->set_index(device_spec.index); + device_spec_proto->set_display_name(video_cards[device_spec.index].display_name); + break; + case InputSourceType::ALSA_INPUT: + alsa_pool.serialize_device(device_spec.index, device_spec_proto); + break; + case InputSourceType::FFMPEG_VIDEO_INPUT: + device_spec_proto->set_type(DeviceSpecProto::FFMPEG_VIDEO_INPUT); + device_spec_proto->set_index(device_spec.index); + device_spec_proto->set_display_name(ffmpeg_inputs[device_spec.index].display_name); + break; + } +} + +void AudioMixer::set_simple_input(unsigned card_index) +{ + assert(card_index < num_capture_cards + num_ffmpeg_inputs); + InputMapping new_input_mapping; + InputMapping::Bus input; + input.name = "Main"; + if (card_index >= num_capture_cards) { + input.device = DeviceSpec{InputSourceType::FFMPEG_VIDEO_INPUT, card_index - num_capture_cards}; + } else { + input.device = DeviceSpec{InputSourceType::CAPTURE_CARD, card_index}; + } + input.source_channel[0] = 0; + input.source_channel[1] = 1; + + new_input_mapping.buses.push_back(input); + + lock_guard lock(audio_mutex); + current_mapping_mode = MappingMode::SIMPLE; + set_input_mapping_lock_held(new_input_mapping); + fader_volume_db[0] = 0.0f; +} + +unsigned AudioMixer::get_simple_input() const +{ + lock_guard lock(audio_mutex); + if (input_mapping.buses.size() == 1 && + input_mapping.buses[0].device.type == InputSourceType::CAPTURE_CARD && + input_mapping.buses[0].source_channel[0] == 0 && + input_mapping.buses[0].source_channel[1] == 1) { + return input_mapping.buses[0].device.index; + } else if (input_mapping.buses.size() == 1 && + input_mapping.buses[0].device.type == InputSourceType::FFMPEG_VIDEO_INPUT && + input_mapping.buses[0].source_channel[0] == 0 && + input_mapping.buses[0].source_channel[1] == 1) { + return input_mapping.buses[0].device.index + num_capture_cards; + } else { + return numeric_limits::max(); + } +} + +void AudioMixer::set_input_mapping(const InputMapping &new_input_mapping) +{ + lock_guard lock(audio_mutex); + set_input_mapping_lock_held(new_input_mapping); + current_mapping_mode = MappingMode::MULTICHANNEL; +} + +AudioMixer::MappingMode AudioMixer::get_mapping_mode() const +{ + lock_guard lock(audio_mutex); + return current_mapping_mode; +} + +void AudioMixer::set_input_mapping_lock_held(const InputMapping &new_input_mapping) +{ + map> interesting_channels; + for (const InputMapping::Bus &bus : new_input_mapping.buses) { + if (bus.device.type == InputSourceType::CAPTURE_CARD || + bus.device.type == InputSourceType::ALSA_INPUT || + bus.device.type == InputSourceType::FFMPEG_VIDEO_INPUT) { + for (unsigned channel = 0; channel < 2; ++channel) { + if (bus.source_channel[channel] != -1) { + interesting_channels[bus.device].insert(bus.source_channel[channel]); + } + } + } else { + assert(bus.device.type == InputSourceType::SILENCE); + } + } + + // Kill all the old metrics, and set up new ones. + for (unsigned bus_index = 0; bus_index < input_mapping.buses.size(); ++bus_index) { + BusMetrics &metrics = bus_metrics[bus_index]; + + vector> labels_left = metrics.labels; + labels_left.emplace_back("channel", "left"); + vector> labels_right = metrics.labels; + labels_right.emplace_back("channel", "right"); + + global_metrics.remove("bus_current_level_dbfs", labels_left); + global_metrics.remove("bus_current_level_dbfs", labels_right); + global_metrics.remove("bus_peak_level_dbfs", labels_left); + global_metrics.remove("bus_peak_level_dbfs", labels_right); + global_metrics.remove("bus_historic_peak_dbfs", metrics.labels); + global_metrics.remove("bus_gain_staging_db", metrics.labels); + global_metrics.remove("bus_compressor_attenuation_db", metrics.labels); + } + bus_metrics.reset(new BusMetrics[new_input_mapping.buses.size()]); + for (unsigned bus_index = 0; bus_index < new_input_mapping.buses.size(); ++bus_index) { + const InputMapping::Bus &bus = new_input_mapping.buses[bus_index]; + BusMetrics &metrics = bus_metrics[bus_index]; + + char bus_index_str[16], source_index_str[16], source_channels_str[64]; + snprintf(bus_index_str, sizeof(bus_index_str), "%u", bus_index); + snprintf(source_index_str, sizeof(source_index_str), "%u", bus.device.index); + snprintf(source_channels_str, sizeof(source_channels_str), "%d:%d", bus.source_channel[0], bus.source_channel[1]); + + vector> labels; + metrics.labels.emplace_back("index", bus_index_str); + metrics.labels.emplace_back("name", bus.name); + if (bus.device.type == InputSourceType::SILENCE) { + metrics.labels.emplace_back("source_type", "silence"); + } else if (bus.device.type == InputSourceType::CAPTURE_CARD) { + metrics.labels.emplace_back("source_type", "capture_card"); + } else if (bus.device.type == InputSourceType::ALSA_INPUT) { + metrics.labels.emplace_back("source_type", "alsa_input"); + } else if (bus.device.type == InputSourceType::FFMPEG_VIDEO_INPUT) { + metrics.labels.emplace_back("source_type", "ffmpeg_video_input"); + } else { + assert(false); + } + metrics.labels.emplace_back("source_index", source_index_str); + metrics.labels.emplace_back("source_channels", source_channels_str); + + vector> labels_left = metrics.labels; + labels_left.emplace_back("channel", "left"); + vector> labels_right = metrics.labels; + labels_right.emplace_back("channel", "right"); + + global_metrics.add("bus_current_level_dbfs", labels_left, &metrics.current_level_dbfs[0], Metrics::TYPE_GAUGE); + global_metrics.add("bus_current_level_dbfs", labels_right, &metrics.current_level_dbfs[1], Metrics::TYPE_GAUGE); + global_metrics.add("bus_peak_level_dbfs", labels_left, &metrics.peak_level_dbfs[0], Metrics::TYPE_GAUGE); + global_metrics.add("bus_peak_level_dbfs", labels_right, &metrics.peak_level_dbfs[1], Metrics::TYPE_GAUGE); + global_metrics.add("bus_historic_peak_dbfs", metrics.labels, &metrics.historic_peak_dbfs, Metrics::TYPE_GAUGE); + global_metrics.add("bus_gain_staging_db", metrics.labels, &metrics.gain_staging_db, Metrics::TYPE_GAUGE); + global_metrics.add("bus_compressor_attenuation_db", metrics.labels, &metrics.compressor_attenuation_db, Metrics::TYPE_GAUGE); + } + + // Reset resamplers for all cards that don't have the exact same state as before. + for (unsigned card_index = 0; card_index < MAX_VIDEO_CARDS; ++card_index) { + const DeviceSpec device_spec{InputSourceType::CAPTURE_CARD, card_index}; + AudioDevice *device = find_audio_device(device_spec); + if (device->interesting_channels != interesting_channels[device_spec]) { + device->interesting_channels = interesting_channels[device_spec]; + reset_resampler_mutex_held(device_spec); + } + } + for (unsigned card_index = 0; card_index < MAX_ALSA_CARDS; ++card_index) { + const DeviceSpec device_spec{InputSourceType::ALSA_INPUT, card_index}; + AudioDevice *device = find_audio_device(device_spec); + if (interesting_channels[device_spec].empty()) { + alsa_pool.release_device(card_index); + } else { + alsa_pool.hold_device(card_index); + } + if (device->interesting_channels != interesting_channels[device_spec]) { + device->interesting_channels = interesting_channels[device_spec]; + alsa_pool.reset_device(device_spec.index); + reset_resampler_mutex_held(device_spec); + } + } + for (unsigned card_index = 0; card_index < num_ffmpeg_inputs; ++card_index) { + const DeviceSpec device_spec{InputSourceType::FFMPEG_VIDEO_INPUT, card_index}; + AudioDevice *device = find_audio_device(device_spec); + if (device->interesting_channels != interesting_channels[device_spec]) { + device->interesting_channels = interesting_channels[device_spec]; + reset_resampler_mutex_held(device_spec); + } + } + + input_mapping = new_input_mapping; +} + +InputMapping AudioMixer::get_input_mapping() const +{ + lock_guard lock(audio_mutex); + return input_mapping; +} + +unsigned AudioMixer::num_buses() const +{ + lock_guard lock(audio_mutex); + return input_mapping.buses.size(); +} + +void AudioMixer::reset_peak(unsigned bus_index) +{ + lock_guard lock(audio_mutex); + for (unsigned channel = 0; channel < 2; ++channel) { + PeakHistory &history = peak_history[bus_index][channel]; + history.current_level = 0.0f; + history.historic_peak = 0.0f; + history.current_peak = 0.0f; + history.last_peak = 0.0f; + history.age_seconds = 0.0f; + } +} + +bool AudioMixer::is_mono(unsigned bus_index) +{ + lock_guard lock(audio_mutex); + const InputMapping::Bus &bus = input_mapping.buses[bus_index]; + if (bus.device.type == InputSourceType::SILENCE) { + return true; + } else { + assert(bus.device.type == InputSourceType::CAPTURE_CARD || + bus.device.type == InputSourceType::ALSA_INPUT || + bus.device.type == InputSourceType::FFMPEG_VIDEO_INPUT); + return bus.source_channel[0] == bus.source_channel[1]; + } +} + +AudioMixer *global_audio_mixer = nullptr; diff --git a/nageru/audio_mixer.h b/nageru/audio_mixer.h new file mode 100644 index 0000000..9793646 --- /dev/null +++ b/nageru/audio_mixer.h @@ -0,0 +1,429 @@ +#ifndef _AUDIO_MIXER_H +#define _AUDIO_MIXER_H 1 + +// The audio mixer, dealing with extracting the right signals from +// each capture card, resampling signals so that they are in sync, +// processing them with effects (if desired), and then mixing them +// all together into one final audio signal. +// +// All operations on AudioMixer (except destruction) are thread-safe. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "alsa_pool.h" +#include "correlation_measurer.h" +#include "db.h" +#include "defs.h" +#include "ebu_r128_proc.h" +#include "filter.h" +#include "input_mapping.h" +#include "resampling_queue.h" +#include "stereocompressor.h" + +class DeviceSpecProto; + +namespace bmusb { +struct AudioFormat; +} // namespace bmusb + +enum EQBand { + EQ_BAND_BASS = 0, + EQ_BAND_MID, + EQ_BAND_TREBLE, + NUM_EQ_BANDS +}; + +class AudioMixer { +public: + AudioMixer(unsigned num_capture_cards, unsigned num_ffmpeg_inputs); + void reset_resampler(DeviceSpec device_spec); + void reset_meters(); + + // Add audio (or silence) to the given device's queue. Can return false if + // the lock wasn't successfully taken; if so, you should simply try again. + // (This is to avoid a deadlock where a card hangs on the mutex in add_audio() + // while we are trying to shut it down from another thread that also holds + // the mutex.) frame_length is in TIMEBASE units. + bool add_audio(DeviceSpec device_spec, const uint8_t *data, unsigned num_samples, bmusb::AudioFormat audio_format, int64_t frame_length, std::chrono::steady_clock::time_point frame_time); + bool add_silence(DeviceSpec device_spec, unsigned samples_per_frame, unsigned num_frames, int64_t frame_length); + + // If a given device is offline for whatever reason and cannot deliver audio + // (by means of add_audio() or add_silence()), you can call put it in silence mode, + // where it will be taken to only output silence. Note that when taking it _out_ + // of silence mode, the resampler will be reset, so that old audio will not + // affect it. Same true/false behavior as add_audio(). + bool silence_card(DeviceSpec device_spec, bool silence); + + std::vector get_output(std::chrono::steady_clock::time_point ts, unsigned num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy); + + float get_fader_volume(unsigned bus_index) const { return fader_volume_db[bus_index]; } + void set_fader_volume(unsigned bus_index, float level_db) { fader_volume_db[bus_index] = level_db; } + + bool get_mute(unsigned bus_index) const { return mute[bus_index]; } + void set_mute(unsigned bus_index, bool muted) { mute[bus_index] = muted; } + + // Note: This operation holds all ALSA devices (see ALSAPool::get_devices()). + // You will need to call set_input_mapping() to get the hold state correctly, + // or every card will be held forever. + std::map get_devices(); + + // See comments on ALSAPool::get_card_state(). + ALSAPool::Device::State get_alsa_card_state(unsigned index) + { + return alsa_pool.get_card_state(index); + } + + // See comments on ALSAPool::create_dead_card(). + DeviceSpec create_dead_card(const std::string &name, const std::string &info, unsigned num_channels) + { + unsigned dead_card_index = alsa_pool.create_dead_card(name, info, num_channels); + return DeviceSpec{InputSourceType::ALSA_INPUT, dead_card_index}; + } + + void set_display_name(DeviceSpec device_spec, const std::string &name); + + // Note: The card should be held (currently this isn't enforced, though). + void serialize_device(DeviceSpec device_spec, DeviceSpecProto *device_spec_proto); + + enum class MappingMode { + // A single bus, only from a video card (no ALSA devices), + // only channel 1 and 2, locked to +0 dB. Note that this is + // only an UI abstraction around exactly the same audio code + // as MULTICHANNEL; it's just less flexible. + SIMPLE, + + // Full, arbitrary mappings. + MULTICHANNEL + }; + + // Automatically sets mapping mode to MappingMode::SIMPLE. + void set_simple_input(unsigned card_index); + + // If mapping mode is not representable as a MappingMode::SIMPLE type + // mapping, returns numeric_limits::max(). + unsigned get_simple_input() const; + + // Implicitly sets mapping mode to MappingMode::MULTICHANNEL. + void set_input_mapping(const InputMapping &input_mapping); + + MappingMode get_mapping_mode() const; + InputMapping get_input_mapping() const; + + unsigned num_buses() const; + + void set_locut_cutoff(float cutoff_hz) + { + locut_cutoff_hz = cutoff_hz; + } + + float get_locut_cutoff() const + { + return locut_cutoff_hz; + } + + void set_locut_enabled(unsigned bus, bool enabled) + { + locut_enabled[bus] = enabled; + } + + bool get_locut_enabled(unsigned bus) + { + return locut_enabled[bus]; + } + + bool is_mono(unsigned bus_index); + + void set_stereo_width(unsigned bus_index, float width) + { + stereo_width[bus_index] = width; + } + + float get_stereo_width(unsigned bus_index) + { + return stereo_width[bus_index]; + } + + void set_eq(unsigned bus_index, EQBand band, float db_gain) + { + assert(band >= 0 && band < NUM_EQ_BANDS); + eq_level_db[bus_index][band] = db_gain; + } + + float get_eq(unsigned bus_index, EQBand band) const + { + assert(band >= 0 && band < NUM_EQ_BANDS); + return eq_level_db[bus_index][band]; + } + + float get_limiter_threshold_dbfs() const + { + return limiter_threshold_dbfs; + } + + float get_compressor_threshold_dbfs(unsigned bus_index) const + { + return compressor_threshold_dbfs[bus_index]; + } + + void set_limiter_threshold_dbfs(float threshold_dbfs) + { + limiter_threshold_dbfs = threshold_dbfs; + } + + void set_compressor_threshold_dbfs(unsigned bus_index, float threshold_dbfs) + { + compressor_threshold_dbfs[bus_index] = threshold_dbfs; + } + + void set_limiter_enabled(bool enabled) + { + limiter_enabled = enabled; + } + + bool get_limiter_enabled() const + { + return limiter_enabled; + } + + void set_compressor_enabled(unsigned bus_index, bool enabled) + { + compressor_enabled[bus_index] = enabled; + } + + bool get_compressor_enabled(unsigned bus_index) const + { + return compressor_enabled[bus_index]; + } + + void set_gain_staging_db(unsigned bus_index, float gain_db) + { + std::unique_lock lock(compressor_mutex); + level_compressor_enabled[bus_index] = false; + gain_staging_db[bus_index] = gain_db; + } + + float get_gain_staging_db(unsigned bus_index) const + { + std::unique_lock lock(compressor_mutex); + return gain_staging_db[bus_index]; + } + + void set_gain_staging_auto(unsigned bus_index, bool enabled) + { + std::unique_lock lock(compressor_mutex); + level_compressor_enabled[bus_index] = enabled; + } + + bool get_gain_staging_auto(unsigned bus_index) const + { + std::unique_lock lock(compressor_mutex); + return level_compressor_enabled[bus_index]; + } + + void set_final_makeup_gain_db(float gain_db) + { + std::unique_lock lock(compressor_mutex); + final_makeup_gain_auto = false; + final_makeup_gain = from_db(gain_db); + } + + float get_final_makeup_gain_db() + { + std::unique_lock lock(compressor_mutex); + return to_db(final_makeup_gain); + } + + void set_final_makeup_gain_auto(bool enabled) + { + std::unique_lock lock(compressor_mutex); + final_makeup_gain_auto = enabled; + } + + bool get_final_makeup_gain_auto() const + { + std::unique_lock lock(compressor_mutex); + return final_makeup_gain_auto; + } + + void reset_peak(unsigned bus_index); + + struct BusLevel { + float current_level_dbfs[2]; // Digital peak of last frame, left and right. + float peak_level_dbfs[2]; // Digital peak with hold, left and right. + float historic_peak_dbfs; + float gain_staging_db; + float compressor_attenuation_db; // A positive number; 0.0 for no attenuation. + }; + + typedef std::function bus_levels, + float global_level_lufs, float range_low_lufs, float range_high_lufs, + float final_makeup_gain_db, + float correlation)> audio_level_callback_t; + void set_audio_level_callback(audio_level_callback_t callback) + { + audio_level_callback = callback; + } + + typedef std::function state_changed_callback_t; + void set_state_changed_callback(state_changed_callback_t callback) + { + state_changed_callback = callback; + } + + state_changed_callback_t get_state_changed_callback() const + { + return state_changed_callback; + } + + void trigger_state_changed_callback() + { + if (state_changed_callback != nullptr) { + state_changed_callback(); + } + } + + // A combination of all settings for a bus. Useful if you want to get + // or store them as a whole without bothering to call all of the get_* + // or set_* functions for that bus. + struct BusSettings { + float fader_volume_db; + bool muted; + bool locut_enabled; + float stereo_width; + float eq_level_db[NUM_EQ_BANDS]; + float gain_staging_db; + bool level_compressor_enabled; + float compressor_threshold_dbfs; + bool compressor_enabled; + }; + static BusSettings get_default_bus_settings(); + BusSettings get_bus_settings(unsigned bus_index) const; + void set_bus_settings(unsigned bus_index, const BusSettings &settings); + +private: + struct AudioDevice { + std::unique_ptr resampling_queue; + std::string display_name; + unsigned capture_frequency = OUTPUT_FREQUENCY; + // Which channels we consider interesting (ie., are part of some input_mapping). + std::set interesting_channels; + bool silenced = false; + }; + + const AudioDevice *find_audio_device(DeviceSpec device_spec) const + { + return const_cast(this)->find_audio_device(device_spec); + } + + AudioDevice *find_audio_device(DeviceSpec device_spec); + + void find_sample_src_from_device(const std::map> &samples_card, DeviceSpec device_spec, int source_channel, const float **srcptr, unsigned *stride); + void fill_audio_bus(const std::map> &samples_card, const InputMapping::Bus &bus, unsigned num_samples, float stereo_width, float *output); + void reset_resampler_mutex_held(DeviceSpec device_spec); + void apply_eq(unsigned bus_index, std::vector *samples_bus); + void update_meters(const std::vector &samples); + void add_bus_to_master(unsigned bus_index, const std::vector &samples_bus, std::vector *samples_out); + void measure_bus_levels(unsigned bus_index, const std::vector &left, const std::vector &right); + void send_audio_level_callback(); + std::vector get_active_devices() const; + void set_input_mapping_lock_held(const InputMapping &input_mapping); + + unsigned num_capture_cards, num_ffmpeg_inputs; + + mutable std::timed_mutex audio_mutex; + + ALSAPool alsa_pool; + AudioDevice video_cards[MAX_VIDEO_CARDS]; // Under audio_mutex. + AudioDevice alsa_inputs[MAX_ALSA_CARDS]; // Under audio_mutex. + std::unique_ptr ffmpeg_inputs; // Under audio_mutex. + + std::atomic locut_cutoff_hz{120}; + StereoFilter locut[MAX_BUSES]; // Default cutoff 120 Hz, 24 dB/oct. + std::atomic locut_enabled[MAX_BUSES]; + StereoFilter eq[MAX_BUSES][NUM_EQ_BANDS]; // The one for EQBand::MID isn't actually used (see comments in apply_eq()). + + // First compressor; takes us up to about -12 dBFS. + mutable std::mutex compressor_mutex; + std::unique_ptr level_compressor[MAX_BUSES]; // Under compressor_mutex. Used to set/override gain_staging_db if . + float gain_staging_db[MAX_BUSES]; // Under compressor_mutex. + float last_gain_staging_db[MAX_BUSES]; // Under compressor_mutex. + bool level_compressor_enabled[MAX_BUSES]; // Under compressor_mutex. + + static constexpr float ref_level_dbfs = -14.0f; // Chosen so that we end up around 0 LU in practice. + static constexpr float ref_level_lufs = -23.0f; // 0 LU, more or less by definition. + + StereoCompressor limiter; + std::atomic limiter_threshold_dbfs{ref_level_dbfs + 4.0f}; // 4 dB. + std::atomic limiter_enabled{true}; + std::unique_ptr compressor[MAX_BUSES]; + std::atomic compressor_threshold_dbfs[MAX_BUSES]; + std::atomic compressor_enabled[MAX_BUSES]; + + // Note: The values here are not in dB. + struct PeakHistory { + float current_level = 0.0f; // Peak of the last frame. + float historic_peak = 0.0f; // Highest peak since last reset; no falloff. + float current_peak = 0.0f; // Current peak of the peak meter. + float last_peak = 0.0f; + float age_seconds = 0.0f; // Time since "last_peak" was set. + }; + PeakHistory peak_history[MAX_BUSES][2]; // Separate for each channel. Under audio_mutex. + + double final_makeup_gain = 1.0; // Under compressor_mutex. Read/write by the user. Note: Not in dB, we want the numeric precision so that we can change it slowly. + bool final_makeup_gain_auto = true; // Under compressor_mutex. + + MappingMode current_mapping_mode; // Under audio_mutex. + InputMapping input_mapping; // Under audio_mutex. + std::atomic fader_volume_db[MAX_BUSES] {{ 0.0f }}; + std::atomic mute[MAX_BUSES] {{ false }}; + float last_fader_volume_db[MAX_BUSES] { 0.0f }; // Under audio_mutex. + std::atomic stereo_width[MAX_BUSES] {{ 0.0f }}; // Default 1.0f (is set in constructor). + std::atomic eq_level_db[MAX_BUSES][NUM_EQ_BANDS] {{{ 0.0f }}}; + float last_eq_level_db[MAX_BUSES][NUM_EQ_BANDS] {{ 0.0f }}; + + audio_level_callback_t audio_level_callback = nullptr; + state_changed_callback_t state_changed_callback = nullptr; + mutable std::mutex audio_measure_mutex; + Ebu_r128_proc r128; // Under audio_measure_mutex. + CorrelationMeasurer correlation; // Under audio_measure_mutex. + Resampler peak_resampler; // Under audio_measure_mutex. + std::atomic peak{0.0f}; + + // Metrics. + std::atomic metric_audio_loudness_short_lufs{0.0 / 0.0}; + std::atomic metric_audio_loudness_integrated_lufs{0.0 / 0.0}; + std::atomic metric_audio_loudness_range_low_lufs{0.0 / 0.0}; + std::atomic metric_audio_loudness_range_high_lufs{0.0 / 0.0}; + std::atomic metric_audio_peak_dbfs{0.0 / 0.0}; + std::atomic metric_audio_final_makeup_gain_db{0.0}; + std::atomic metric_audio_correlation{0.0}; + + // These are all gauges corresponding to the elements of BusLevel. + // In a sense, they'd probably do better as histograms, but that's an + // awful lot of time series when you have many buses. + struct BusMetrics { + std::vector> labels; + std::atomic current_level_dbfs[2]{{0.0/0.0},{0.0/0.0}}; + std::atomic peak_level_dbfs[2]{{0.0/0.0},{0.0/0.0}}; + std::atomic historic_peak_dbfs{0.0/0.0}; + std::atomic gain_staging_db{0.0/0.0}; + std::atomic compressor_attenuation_db{0.0/0.0}; + }; + std::unique_ptr bus_metrics; // One for each bus in . +}; + +extern AudioMixer *global_audio_mixer; + +#endif // !defined(_AUDIO_MIXER_H) diff --git a/nageru/basic_stats.cpp b/nageru/basic_stats.cpp new file mode 100644 index 0000000..937e302 --- /dev/null +++ b/nageru/basic_stats.cpp @@ -0,0 +1,156 @@ +#include "basic_stats.h" +#include "metrics.h" + +#include +#include +#include + +// Epoxy seems to be missing these. Taken from the NVX_gpu_memory_info spec. +#ifndef GPU_MEMORY_INFO_DEDICATED_VIDMEM_NVX +#define GPU_MEMORY_INFO_DEDICATED_VIDMEM_NVX 0x9047 +#endif +#ifndef GPU_MEMORY_INFO_TOTAL_AVAILABLE_MEMORY_NVX +#define GPU_MEMORY_INFO_TOTAL_AVAILABLE_MEMORY_NVX 0x9048 +#endif +#ifndef GPU_MEMORY_INFO_CURRENT_AVAILABLE_VIDMEM_NVX +#define GPU_MEMORY_INFO_CURRENT_AVAILABLE_VIDMEM_NVX 0x9049 +#endif +#ifndef GPU_MEMORY_INFO_EVICTION_COUNT_NVX +#define GPU_MEMORY_INFO_EVICTION_COUNT_NVX 0x904A +#endif +#ifndef GPU_MEMORY_INFO_EVICTED_MEMORY_NVX +#define GPU_MEMORY_INFO_EVICTED_MEMORY_NVX 0x904B +#endif + +using namespace std; +using namespace std::chrono; + +bool uses_mlock = false; + +BasicStats::BasicStats(bool verbose, bool use_opengl) + : verbose(verbose) +{ + start = steady_clock::now(); + + metric_start_time_seconds = get_timestamp_for_metrics(); + global_metrics.add("frames_output_total", &metric_frames_output_total); + global_metrics.add("frames_output_dropped", &metric_frames_output_dropped); + global_metrics.add("start_time_seconds", &metric_start_time_seconds, Metrics::TYPE_GAUGE); + global_metrics.add("memory_used_bytes", &metrics_memory_used_bytes); + global_metrics.add("memory_locked_limit_bytes", &metrics_memory_locked_limit_bytes); + + // TODO: It would be nice to compile this out entirely for Kaeru, + // to avoid pulling in the symbols from libGL/Epoxy. + if (use_opengl) { + gpu_memory_stats.reset(new GPUMemoryStats(verbose)); + } +} + +void BasicStats::update(int frame_num, int stats_dropped_frames) +{ + steady_clock::time_point now = steady_clock::now(); + double elapsed = duration(now - start).count(); + + metric_frames_output_total = frame_num; + metric_frames_output_dropped = stats_dropped_frames; + + if (frame_num % 100 != 0) { + return; + } + + if (verbose) { + printf("%d frames (%d dropped) in %.3f seconds = %.1f fps (%.1f ms/frame)", + frame_num, stats_dropped_frames, elapsed, frame_num / elapsed, + 1e3 * elapsed / frame_num); + } + + // Check our memory usage, to see if we are close to our mlockall() + // limit (if at all set). + rusage used; + if (getrusage(RUSAGE_SELF, &used) == -1) { + perror("getrusage(RUSAGE_SELF)"); + assert(false); + } + metrics_memory_used_bytes = used.ru_maxrss * 1024; + + if (uses_mlock) { + rlimit limit; + if (getrlimit(RLIMIT_MEMLOCK, &limit) == -1) { + perror("getrlimit(RLIMIT_MEMLOCK)"); + assert(false); + } + metrics_memory_locked_limit_bytes = limit.rlim_cur; + + if (verbose) { + if (limit.rlim_cur == 0) { + printf(", using %ld MB memory (locked)", + long(used.ru_maxrss / 1024)); + } else { + printf(", using %ld / %ld MB lockable memory (%.1f%%)", + long(used.ru_maxrss / 1024), + long(limit.rlim_cur / 1048576), + float(100.0 * (used.ru_maxrss * 1024.0) / limit.rlim_cur)); + } + } + } else { + metrics_memory_locked_limit_bytes = 0.0 / 0.0; + if (verbose) { + printf(", using %ld MB memory (not locked)", + long(used.ru_maxrss / 1024)); + } + } + + if (gpu_memory_stats != nullptr) { + gpu_memory_stats->update(); + } + + if (verbose) { + printf("\n"); + } +} + +GPUMemoryStats::GPUMemoryStats(bool verbose) + : verbose(verbose) +{ + // GL_NV_query_memory is exposed but supposedly only works on + // Quadro/Titan cards, so we use GL_NVX_gpu_memory_info even though it's + // formally marked as experimental. + // Intel/Mesa doesn't seem to have anything comparable (at least nothing + // that gets the amount of _available_ memory). + supported = epoxy_has_gl_extension("GL_NVX_gpu_memory_info"); + if (supported) { + global_metrics.add("memory_gpu_total_bytes", &metric_memory_gpu_total_bytes, Metrics::TYPE_GAUGE); + global_metrics.add("memory_gpu_dedicated_bytes", &metric_memory_gpu_dedicated_bytes, Metrics::TYPE_GAUGE); + global_metrics.add("memory_gpu_used_bytes", &metric_memory_gpu_used_bytes, Metrics::TYPE_GAUGE); + global_metrics.add("memory_gpu_evicted_bytes", &metric_memory_gpu_evicted_bytes, Metrics::TYPE_GAUGE); + global_metrics.add("memory_gpu_evictions", &metric_memory_gpu_evictions); + } +} + +void GPUMemoryStats::update() +{ + if (!supported) { + return; + } + + GLint total, dedicated, available, evicted, evictions; + glGetIntegerv(GPU_MEMORY_INFO_TOTAL_AVAILABLE_MEMORY_NVX, &total); + glGetIntegerv(GPU_MEMORY_INFO_DEDICATED_VIDMEM_NVX, &dedicated); + glGetIntegerv(GPU_MEMORY_INFO_CURRENT_AVAILABLE_VIDMEM_NVX, &available); + glGetIntegerv(GPU_MEMORY_INFO_EVICTED_MEMORY_NVX, &evicted); + glGetIntegerv(GPU_MEMORY_INFO_EVICTION_COUNT_NVX, &evictions); + + if (glGetError() == 0) { + metric_memory_gpu_total_bytes = int64_t(total) * 1024; + metric_memory_gpu_dedicated_bytes = int64_t(dedicated) * 1024; + metric_memory_gpu_used_bytes = int64_t(total - available) * 1024; + metric_memory_gpu_evicted_bytes = int64_t(evicted) * 1024; + metric_memory_gpu_evictions = evictions; + + if (verbose) { + printf(", using %d / %d MB GPU memory (%.1f%%)", + (total - available) / 1024, total / 1024, + float(100.0 * (total - available) / total)); + } + } +} diff --git a/nageru/basic_stats.h b/nageru/basic_stats.h new file mode 100644 index 0000000..bff7881 --- /dev/null +++ b/nageru/basic_stats.h @@ -0,0 +1,53 @@ +#ifndef _BASIC_STATS_H +#define _BASIC_STATS_H + +// Holds some metrics for basic statistics about uptime, memory usage and such. + +#include + +#include +#include +#include + +extern bool uses_mlock; + +class GPUMemoryStats; + +class BasicStats { +public: + BasicStats(bool verbose, bool use_opengl); + void update(int frame_num, int stats_dropped_frames); + +private: + std::chrono::steady_clock::time_point start; + bool verbose; + std::unique_ptr gpu_memory_stats; + + // Metrics. + std::atomic metric_frames_output_total{0}; + std::atomic metric_frames_output_dropped{0}; + std::atomic metric_start_time_seconds{0.0 / 0.0}; + std::atomic metrics_memory_used_bytes{0}; + std::atomic metrics_memory_locked_limit_bytes{0.0 / 0.0}; +}; + +// Holds some metrics for GPU memory usage. Currently only exposed for NVIDIA cards +// (no-op on all other platforms). + +class GPUMemoryStats { +public: + GPUMemoryStats(bool verbose); + void update(); + +private: + bool verbose, supported; + + // Metrics. + std::atomic metric_memory_gpu_total_bytes{0}; + std::atomic metric_memory_gpu_dedicated_bytes{0}; + std::atomic metric_memory_gpu_used_bytes{0}; + std::atomic metric_memory_gpu_evicted_bytes{0}; + std::atomic metric_memory_gpu_evictions{0}; +}; + +#endif // !defined(_BASIC_STATS_H) diff --git a/nageru/benchmark_audio_mixer.cpp b/nageru/benchmark_audio_mixer.cpp new file mode 100644 index 0000000..b47c340 --- /dev/null +++ b/nageru/benchmark_audio_mixer.cpp @@ -0,0 +1,185 @@ +// Rather simplistic benchmark of AudioMixer. Sets up a simple mapping +// with the default settings, feeds some white noise to the inputs and +// runs a while. Useful for e.g. profiling. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "audio_mixer.h" +#include "db.h" +#include "defs.h" +#include "input_mapping.h" +#include "resampling_queue.h" +#include "timebase.h" + +#define NUM_BENCHMARK_CARDS 4 +#define NUM_WARMUP_FRAMES 100 +#define NUM_BENCHMARK_FRAMES 1000 +#define NUM_TEST_FRAMES 10 +#define NUM_CHANNELS 8 +#define NUM_SAMPLES 1024 + +using namespace std; +using namespace std::chrono; + +// 16-bit samples, white noise at full volume. +uint8_t samples16[(NUM_SAMPLES * NUM_CHANNELS + 1024) * sizeof(uint16_t)]; + +// 24-bit samples, white noise at low volume (-48 dB). +uint8_t samples24[(NUM_SAMPLES * NUM_CHANNELS + 1024) * 3]; + +static uint32_t seed = 1234; + +// We use our own instead of rand() to get deterministic behavior. +// Quality doesn't really matter much. +uint32_t lcgrand() +{ + seed = seed * 1103515245u + 12345u; + return seed; +} + +void reset_lcgrand() +{ + seed = 1234; +} + +void callback(float level_lufs, float peak_db, + std::vector bus_levels, + float global_level_lufs, float range_low_lufs, float range_high_lufs, + float final_makeup_gain_db, + float correlation) +{ + // Empty. +} + +vector process_frame(unsigned frame_num, AudioMixer *mixer) +{ + duration> frame_duration(frame_num); + steady_clock::time_point ts = steady_clock::time_point(duration_cast(frame_duration)); + + // Feed the inputs. + for (unsigned card_index = 0; card_index < NUM_BENCHMARK_CARDS; ++card_index) { + bmusb::AudioFormat audio_format; + audio_format.bits_per_sample = card_index == 3 ? 24 : 16; + audio_format.num_channels = NUM_CHANNELS; + + unsigned num_samples = NUM_SAMPLES + (lcgrand() % 9) - 5; + bool ok = mixer->add_audio(DeviceSpec{InputSourceType::CAPTURE_CARD, card_index}, + card_index == 3 ? samples24 : samples16, num_samples, audio_format, + NUM_SAMPLES * TIMEBASE / OUTPUT_FREQUENCY, ts); + assert(ok); + } + + return mixer->get_output(ts, NUM_SAMPLES, ResamplingQueue::ADJUST_RATE); +} + +void init_mapping(AudioMixer *mixer) +{ + InputMapping mapping; + + InputMapping::Bus bus1; + bus1.device = DeviceSpec{InputSourceType::CAPTURE_CARD, 0}; + bus1.source_channel[0] = 0; + bus1.source_channel[1] = 1; + mapping.buses.push_back(bus1); + + InputMapping::Bus bus2; + bus2.device = DeviceSpec{InputSourceType::CAPTURE_CARD, 3}; + bus2.source_channel[0] = 6; + bus2.source_channel[1] = 4; + mapping.buses.push_back(bus2); + + mixer->set_input_mapping(mapping); +} + +void do_test(const char *filename) +{ + AudioMixer mixer(NUM_BENCHMARK_CARDS, 0); + mixer.set_audio_level_callback(callback); + init_mapping(&mixer); + + reset_lcgrand(); + + vector output; + for (unsigned i = 0; i < NUM_TEST_FRAMES; ++i) { + vector frame_output = process_frame(i, &mixer); + output.insert(output.end(), frame_output.begin(), frame_output.end()); + } + + FILE *fp = fopen(filename, "rb"); + if (fp == nullptr) { + fprintf(stderr, "%s not found, writing new reference.\n", filename); + fp = fopen(filename, "wb"); + fwrite(&output[0], output.size() * sizeof(float), 1, fp); + fclose(fp); + return; + } + + vector ref; + ref.resize(output.size()); + fread(&ref[0], output.size() * sizeof(float), 1, fp); + fclose(fp); + + float max_err = 0.0f, sum_sq_err = 0.0f; + for (unsigned i = 0; i < output.size(); ++i) { + float err = output[i] - ref[i]; + max_err = max(max_err, fabs(err)); + sum_sq_err += err * err; + } + + printf("Largest error: %.6f (%+.1f dB)\n", max_err, to_db(max_err)); + printf("RMS error: %+.1f dB\n", to_db(sqrt(sum_sq_err) / output.size())); +} + +void do_benchmark() +{ + AudioMixer mixer(NUM_BENCHMARK_CARDS, 0); + mixer.set_audio_level_callback(callback); + init_mapping(&mixer); + + size_t out_samples = 0; + + reset_lcgrand(); + + steady_clock::time_point start, end; + for (unsigned i = 0; i < NUM_WARMUP_FRAMES + NUM_BENCHMARK_FRAMES; ++i) { + if (i == NUM_WARMUP_FRAMES) { + start = steady_clock::now(); + } + vector output = process_frame(i, &mixer); + if (i >= NUM_WARMUP_FRAMES) { + out_samples += output.size(); + } + } + end = steady_clock::now(); + + double elapsed = duration(end - start).count(); + double simulated = double(out_samples) / (OUTPUT_FREQUENCY * 2); + printf("%ld samples produced in %.1f ms (%.1f%% CPU, %.1fx realtime).\n", + out_samples, elapsed * 1e3, 100.0 * elapsed / simulated, simulated / elapsed); +} + +int main(int argc, char **argv) +{ + for (unsigned i = 0; i < NUM_SAMPLES * NUM_CHANNELS + 1024; ++i) { + samples16[i * 2] = lcgrand() & 0xff; + samples16[i * 2 + 1] = lcgrand() & 0xff; + + samples24[i * 3] = lcgrand() & 0xff; + samples24[i * 3 + 1] = lcgrand() & 0xff; + samples24[i * 3 + 2] = 0; + } + + if (argc == 2) { + do_test(argv[1]); + } + do_benchmark(); +} + diff --git a/nageru/bg.jpeg b/nageru/bg.jpeg new file mode 100644 index 0000000..268dd58 Binary files /dev/null and b/nageru/bg.jpeg differ diff --git a/nageru/cef_capture.cpp b/nageru/cef_capture.cpp new file mode 100644 index 0000000..b6b8cca --- /dev/null +++ b/nageru/cef_capture.cpp @@ -0,0 +1,264 @@ +#include +#include +#include +#include +#include +#include + +#include "cef_capture.h" +#include "nageru_cef_app.h" + +#undef CHECK +#include +#include +#include + +#include "bmusb/bmusb.h" + +using namespace std; +using namespace std::chrono; +using namespace bmusb; + +extern CefRefPtr cef_app; + +CEFCapture::CEFCapture(const string &url, unsigned width, unsigned height) + : cef_client(new NageruCEFClient(this)), + width(width), + height(height), + start_url(url) +{ + char buf[256]; + snprintf(buf, sizeof(buf), "CEF card %d", card_index + 1); + description = buf; +} + +CEFCapture::~CEFCapture() +{ + if (has_dequeue_callbacks) { + dequeue_cleanup_callback(); + } +} + +void CEFCapture::post_to_cef_ui_thread(std::function &&func, int64_t delay_ms) +{ + lock_guard lock(browser_mutex); + if (browser != nullptr) { + if (delay_ms <= 0) { + CefPostTask(TID_UI, new CEFTaskAdapter(std::move(func))); + } else { + CefPostDelayedTask(TID_UI, new CEFTaskAdapter(std::move(func)), delay_ms); + } + } else { + deferred_tasks.push_back(std::move(func)); + } +} + +void CEFCapture::set_url(const string &url) +{ + post_to_cef_ui_thread([this, url] { + loaded = false; + browser->GetMainFrame()->LoadURL(url); + }); +} + +void CEFCapture::reload() +{ + post_to_cef_ui_thread([this] { + loaded = false; + browser->Reload(); + }); +} + +void CEFCapture::set_max_fps(int max_fps) +{ + post_to_cef_ui_thread([this, max_fps] { + browser->GetHost()->SetWindowlessFrameRate(max_fps); + this->max_fps = max_fps; + }); +} + +void CEFCapture::execute_javascript_async(const string &js) +{ + post_to_cef_ui_thread([this, js] { + if (loaded) { + CefString script_url(""); + int start_line = 1; + browser->GetMainFrame()->ExecuteJavaScript(js, script_url, start_line); + } else { + deferred_javascript.push_back(js); + } + }); +} + +void CEFCapture::resize(unsigned width, unsigned height) +{ + lock_guard lock(resolution_mutex); + this->width = width; + this->height = height; +} + +void CEFCapture::request_new_frame() +{ + // By adding a delay, we make sure we don't get a new frame + // delivered immediately (we probably already are on the UI thread), + // where we couldn't really deal with it. + post_to_cef_ui_thread([this] { + lock_guard lock(browser_mutex); + if (browser != nullptr) { // Could happen if we are shutting down. + browser->GetHost()->Invalidate(PET_VIEW); + } + }, 16); +} + +void CEFCapture::OnPaint(const void *buffer, int width, int height) +{ + steady_clock::time_point timestamp = steady_clock::now(); + + VideoFormat video_format; + video_format.width = width; + video_format.height = height; + video_format.stride = width * 4; + video_format.frame_rate_nom = max_fps; + video_format.frame_rate_den = 1; + video_format.has_signal = true; + video_format.is_connected = true; + + FrameAllocator::Frame video_frame = video_frame_allocator->alloc_frame(); + if (video_frame.data == nullptr) { + // We lost a frame, so we need to invalidate the entire thing. + // (CEF only sends OnPaint when there are actual changes, + // so we need to do this explicitly, or we could be stuck on an + // old frame forever if the image doesn't change.) + request_new_frame(); + ++timecode; + } else { + assert(video_frame.size >= unsigned(width * height * 4)); + assert(!video_frame.interleaved); + memcpy(video_frame.data, buffer, width * height * 4); + video_frame.len = video_format.stride * height; + video_frame.received_timestamp = timestamp; + frame_callback(timecode++, + video_frame, 0, video_format, + FrameAllocator::Frame(), 0, AudioFormat()); + } +} + +void CEFCapture::OnLoadEnd() +{ + post_to_cef_ui_thread([this] { + loaded = true; + for (const string &js : deferred_javascript) { + CefString script_url(""); + int start_line = 1; + browser->GetMainFrame()->ExecuteJavaScript(js, script_url, start_line); + } + deferred_javascript.clear(); + }); +} + +#define FRAME_SIZE (8 << 20) // 8 MB. + +void CEFCapture::configure_card() +{ + if (video_frame_allocator == nullptr) { + owned_video_frame_allocator.reset(new MallocFrameAllocator(FRAME_SIZE, NUM_QUEUED_VIDEO_FRAMES)); + set_video_frame_allocator(owned_video_frame_allocator.get()); + } +} + +void CEFCapture::start_bm_capture() +{ + cef_app->initialize_cef(); + + CefPostTask(TID_UI, new CEFTaskAdapter([this]{ + lock_guard lock(browser_mutex); + + CefBrowserSettings browser_settings; + browser_settings.web_security = cef_state_t::STATE_DISABLED; + browser_settings.webgl = cef_state_t::STATE_ENABLED; + browser_settings.windowless_frame_rate = max_fps; + + CefWindowInfo window_info; + window_info.SetAsWindowless(0); + browser = CefBrowserHost::CreateBrowserSync(window_info, cef_client, start_url, browser_settings, nullptr); + for (function &task : deferred_tasks) { + task(); + } + deferred_tasks.clear(); + })); +} + +void CEFCapture::stop_dequeue_thread() +{ + { + lock_guard lock(browser_mutex); + cef_app->close_browser(browser); + browser = nullptr; // Or unref_cef() will be sad. + } + cef_app->unref_cef(); +} + +std::map CEFCapture::get_available_video_modes() const +{ + VideoMode mode; + + char buf[256]; + snprintf(buf, sizeof(buf), "%ux%u", width, height); + mode.name = buf; + + mode.autodetect = false; + mode.width = width; + mode.height = height; + mode.frame_rate_num = max_fps; + mode.frame_rate_den = 1; + mode.interlaced = false; + + return {{ 0, mode }}; +} + +std::map CEFCapture::get_available_video_inputs() const +{ + return {{ 0, "HTML video input" }}; +} + +std::map CEFCapture::get_available_audio_inputs() const +{ + return {{ 0, "Fake HTML audio input (silence)" }}; +} + +void CEFCapture::set_video_mode(uint32_t video_mode_id) +{ + assert(video_mode_id == 0); +} + +void CEFCapture::set_video_input(uint32_t video_input_id) +{ + assert(video_input_id == 0); +} + +void CEFCapture::set_audio_input(uint32_t audio_input_id) +{ + assert(audio_input_id == 0); +} + +void NageruCEFClient::OnPaint(CefRefPtr browser, PaintElementType type, const RectList &dirtyRects, const void *buffer, int width, int height) +{ + parent->OnPaint(buffer, width, height); +} + +bool NageruCEFClient::GetViewRect(CefRefPtr browser, CefRect &rect) +{ + return parent->GetViewRect(rect); +} + +bool CEFCapture::GetViewRect(CefRect &rect) +{ + lock_guard lock(resolution_mutex); + rect = CefRect(0, 0, width, height); + return true; +} + +void NageruCEFClient::OnLoadEnd(CefRefPtr browser, CefRefPtr frame, int httpStatusCode) +{ + parent->OnLoadEnd(); +} diff --git a/nageru/cef_capture.h b/nageru/cef_capture.h new file mode 100644 index 0000000..29deded --- /dev/null +++ b/nageru/cef_capture.h @@ -0,0 +1,211 @@ +#ifndef _CEF_CAPTURE_H +#define _CEF_CAPTURE_H 1 + +// CEFCapture represents a single CEF virtual capture card (usually, there would only +// be one globally), similar to FFmpegCapture. It owns a CefBrowser, which calls +// OnPaint() back every time it has a frame. Note that it runs asynchronously; +// there's no way to get frame-perfect sync. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#undef CHECK +#include +#include +#include + +#include + +class CefBrowser; +class CefRect; +class CEFCapture; + +// A helper class for CEFCapture to proxy information to CEF, without becoming +// CEF-refcounted itself. +class NageruCEFClient : public CefClient, public CefRenderHandler, public CefLoadHandler +{ +public: + NageruCEFClient(CEFCapture *parent) + : parent(parent) {} + + CefRefPtr GetRenderHandler() override + { + return this; + } + + CefRefPtr GetLoadHandler() override + { + return this; + } + + // CefRenderHandler. + + void OnPaint(CefRefPtr browser, PaintElementType type, const RectList &dirtyRects, const void *buffer, int width, int height) override; + + bool GetViewRect(CefRefPtr browser, CefRect &rect) override; + + // CefLoadHandler. + + void OnLoadEnd(CefRefPtr browser, CefRefPtr frame, int httpStatusCode) override; + +private: + CEFCapture *parent; + + IMPLEMENT_REFCOUNTING(NageruCEFClient); +}; + +class CEFCapture : public bmusb::CaptureInterface +{ +public: + CEFCapture(const std::string &url, unsigned width, unsigned height); + ~CEFCapture(); + + void set_card_index(int card_index) + { + this->card_index = card_index; + } + + int get_card_index() const + { + return card_index; + } + + void set_url(const std::string &url); + void reload(); + void set_max_fps(int max_fps); + void execute_javascript_async(const std::string &js); + void resize(unsigned width, unsigned height); + void request_new_frame(); + + // Callbacks from NageruCEFClient. + void OnPaint(const void *buffer, int width, int height); + bool GetViewRect(CefRect &rect); + void OnLoadEnd(); + + // CaptureInterface. + void set_video_frame_allocator(bmusb::FrameAllocator *allocator) override + { + video_frame_allocator = allocator; + if (owned_video_frame_allocator.get() != allocator) { + owned_video_frame_allocator.reset(); + } + } + + bmusb::FrameAllocator *get_video_frame_allocator() override + { + return video_frame_allocator; + } + + // Does not take ownership. + void set_audio_frame_allocator(bmusb::FrameAllocator *allocator) override + { + } + + bmusb::FrameAllocator *get_audio_frame_allocator() override + { + return nullptr; + } + + void set_frame_callback(bmusb::frame_callback_t callback) override + { + frame_callback = callback; + } + + void set_dequeue_thread_callbacks(std::function init, std::function cleanup) override + { + dequeue_init_callback = init; + dequeue_cleanup_callback = cleanup; + has_dequeue_callbacks = true; + } + + std::string get_description() const override + { + return description; + } + + void configure_card() override; + void start_bm_capture() override; + void stop_dequeue_thread() override; + bool get_disconnected() const override { return false; } + + std::set get_available_pixel_formats() const override + { + return std::set{ bmusb::PixelFormat_8BitBGRA }; + } + + void set_pixel_format(bmusb::PixelFormat pixel_format) override + { + assert(pixel_format == bmusb::PixelFormat_8BitBGRA); + } + + bmusb::PixelFormat get_current_pixel_format() const override + { + return bmusb::PixelFormat_8BitBGRA; + } + + std::map get_available_video_modes() const override; + void set_video_mode(uint32_t video_mode_id) override; + uint32_t get_current_video_mode() const override { return 0; } + + std::map get_available_video_inputs() const override; + void set_video_input(uint32_t video_input_id) override; + uint32_t get_current_video_input() const override { return 0; } + + std::map get_available_audio_inputs() const override; + void set_audio_input(uint32_t audio_input_id) override; + uint32_t get_current_audio_input() const override { return 0; } + +private: + void post_to_cef_ui_thread(std::function &&func, int64_t delay_ms = 0); + + CefRefPtr cef_client; + + // Needs to be different from browser_mutex below, since GetViewRect + // can be called unpredictably from when we are already holding + // . + std::mutex resolution_mutex; + unsigned width, height; // Under . + + int card_index = -1; + + bool has_dequeue_callbacks = false; + std::function dequeue_init_callback = nullptr; + std::function dequeue_cleanup_callback = nullptr; + + bmusb::FrameAllocator *video_frame_allocator = nullptr; + std::unique_ptr owned_video_frame_allocator; + bmusb::frame_callback_t frame_callback = nullptr; + + std::string description, start_url; + std::atomic max_fps{60}; + + // Needs to be recursive because the lambda in OnPaint could cause + // OnPaint itself to be called. + std::recursive_mutex browser_mutex; + CefRefPtr browser; // Under . + + // Tasks waiting for to get ready. Under . + std::vector> deferred_tasks; + + // Whether the last set_url() (includes the implicit one in the constructor) + // has loaded yet. Accessed from the CEF thread only. + bool loaded = false; + + // JavaScript waiting for the first page (well, any page) to have loaded. + // Accessed from the CEF thread only. + std::vector deferred_javascript; + + int timecode = 0; +}; + +#endif // !defined(_CEF_CAPTURE_H) diff --git a/nageru/chroma_subsampler.cpp b/nageru/chroma_subsampler.cpp new file mode 100644 index 0000000..96adef1 --- /dev/null +++ b/nageru/chroma_subsampler.cpp @@ -0,0 +1,452 @@ +#include "chroma_subsampler.h" +#include "v210_converter.h" + +#include + +#include +#include +#include + +using namespace movit; +using namespace std; + +ChromaSubsampler::ChromaSubsampler(ResourcePool *resource_pool) + : resource_pool(resource_pool) +{ + vector frag_shader_outputs; + + // Set up stuff for NV12 conversion. + // + // Note: Due to the horizontally co-sited chroma/luma samples in H.264 + // (chrome position is left for horizontal and center for vertical), + // we need to be a bit careful in our subsampling. A diagram will make + // this clearer, showing some luma and chroma samples: + // + // a b c d + // +---+---+---+---+ + // | | | | | + // | Y | Y | Y | Y | + // | | | | | + // +---+---+---+---+ + // + // +-------+-------+ + // | | | + // | C | C | + // | | | + // +-------+-------+ + // + // Clearly, the rightmost chroma sample here needs to be equivalent to + // b/4 + c/2 + d/4. (We could also implement more sophisticated filters, + // of course, but as long as the upsampling is not going to be equally + // sophisticated, it's probably not worth it.) If we sample once with + // no mipmapping, we get just c, ie., no actual filtering in the + // horizontal direction. (For the vertical direction, we can just + // sample in the middle to get the right filtering.) One could imagine + // we could use mipmapping (assuming we can create mipmaps cheaply), + // but then, what we'd get is this: + // + // (a+b)/2 (c+d)/2 + // +-------+-------+ + // | | | + // | Y | Y | + // | | | + // +-------+-------+ + // + // +-------+-------+ + // | | | + // | C | C | + // | | | + // +-------+-------+ + // + // which ends up sampling equally from a and b, which clearly isn't right. Instead, + // we need to do two (non-mipmapped) chroma samples, both hitting exactly in-between + // source pixels. + // + // Sampling in-between b and c gives us the sample (b+c)/2, and similarly for c and d. + // Taking the average of these gives of (b+c)/4 + (c+d)/4 = b/4 + c/2 + d/4, which is + // exactly what we want. + // + // See also http://www.poynton.com/PDFs/Merging_RGB_and_422.pdf, pages 6–7. + + // Cb/Cr shader. + string cbcr_vert_shader = + "#version 130 \n" + " \n" + "in vec2 position; \n" + "in vec2 texcoord; \n" + "out vec2 tc0, tc1; \n" + "uniform vec2 foo_chroma_offset_0; \n" + "uniform vec2 foo_chroma_offset_1; \n" + " \n" + "void main() \n" + "{ \n" + " // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is: \n" + " // \n" + " // 2.000 0.000 0.000 -1.000 \n" + " // 0.000 2.000 0.000 -1.000 \n" + " // 0.000 0.000 -2.000 -1.000 \n" + " // 0.000 0.000 0.000 1.000 \n" + " gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0); \n" + " vec2 flipped_tc = texcoord; \n" + " tc0 = flipped_tc + foo_chroma_offset_0; \n" + " tc1 = flipped_tc + foo_chroma_offset_1; \n" + "} \n"; + string cbcr_frag_shader = + "#version 130 \n" + "in vec2 tc0, tc1; \n" + "uniform sampler2D cbcr_tex; \n" + "out vec4 FragColor, FragColor2; \n" + "void main() { \n" + " FragColor = 0.5 * (texture(cbcr_tex, tc0) + texture(cbcr_tex, tc1)); \n" + " FragColor2 = FragColor; \n" + "} \n"; + cbcr_program_num = resource_pool->compile_glsl_program(cbcr_vert_shader, cbcr_frag_shader, frag_shader_outputs); + check_error(); + cbcr_chroma_offset_0_location = get_uniform_location(cbcr_program_num, "foo", "chroma_offset_0"); + check_error(); + cbcr_chroma_offset_1_location = get_uniform_location(cbcr_program_num, "foo", "chroma_offset_1"); + check_error(); + + cbcr_texture_sampler_uniform = glGetUniformLocation(cbcr_program_num, "cbcr_tex"); + check_error(); + cbcr_position_attribute_index = glGetAttribLocation(cbcr_program_num, "position"); + check_error(); + cbcr_texcoord_attribute_index = glGetAttribLocation(cbcr_program_num, "texcoord"); + check_error(); + + // Same, for UYVY conversion. + string uyvy_vert_shader = + "#version 130 \n" + " \n" + "in vec2 position; \n" + "in vec2 texcoord; \n" + "out vec2 y_tc0, y_tc1, cbcr_tc0, cbcr_tc1; \n" + "uniform vec2 foo_luma_offset_0; \n" + "uniform vec2 foo_luma_offset_1; \n" + "uniform vec2 foo_chroma_offset_0; \n" + "uniform vec2 foo_chroma_offset_1; \n" + " \n" + "void main() \n" + "{ \n" + " // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is: \n" + " // \n" + " // 2.000 0.000 0.000 -1.000 \n" + " // 0.000 2.000 0.000 -1.000 \n" + " // 0.000 0.000 -2.000 -1.000 \n" + " // 0.000 0.000 0.000 1.000 \n" + " gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0); \n" + " vec2 flipped_tc = texcoord; \n" + " y_tc0 = flipped_tc + foo_luma_offset_0; \n" + " y_tc1 = flipped_tc + foo_luma_offset_1; \n" + " cbcr_tc0 = flipped_tc + foo_chroma_offset_0; \n" + " cbcr_tc1 = flipped_tc + foo_chroma_offset_1; \n" + "} \n"; + string uyvy_frag_shader = + "#version 130 \n" + "in vec2 y_tc0, y_tc1, cbcr_tc0, cbcr_tc1; \n" + "uniform sampler2D y_tex, cbcr_tex; \n" + "out vec4 FragColor; \n" + "void main() { \n" + " float y0 = texture(y_tex, y_tc0).r; \n" + " float y1 = texture(y_tex, y_tc1).r; \n" + " vec2 cbcr0 = texture(cbcr_tex, cbcr_tc0).rg; \n" + " vec2 cbcr1 = texture(cbcr_tex, cbcr_tc1).rg; \n" + " vec2 cbcr = 0.5 * (cbcr0 + cbcr1); \n" + " FragColor = vec4(cbcr.g, y0, cbcr.r, y1); \n" + "} \n"; + + uyvy_program_num = resource_pool->compile_glsl_program(uyvy_vert_shader, uyvy_frag_shader, frag_shader_outputs); + check_error(); + uyvy_luma_offset_0_location = get_uniform_location(uyvy_program_num, "foo", "luma_offset_0"); + check_error(); + uyvy_luma_offset_1_location = get_uniform_location(uyvy_program_num, "foo", "luma_offset_1"); + check_error(); + uyvy_chroma_offset_0_location = get_uniform_location(uyvy_program_num, "foo", "chroma_offset_0"); + check_error(); + uyvy_chroma_offset_1_location = get_uniform_location(uyvy_program_num, "foo", "chroma_offset_1"); + check_error(); + + uyvy_y_texture_sampler_uniform = glGetUniformLocation(uyvy_program_num, "y_tex"); + check_error(); + uyvy_cbcr_texture_sampler_uniform = glGetUniformLocation(uyvy_program_num, "cbcr_tex"); + check_error(); + uyvy_position_attribute_index = glGetAttribLocation(uyvy_program_num, "position"); + check_error(); + uyvy_texcoord_attribute_index = glGetAttribLocation(uyvy_program_num, "texcoord"); + check_error(); + + // Shared between the two. + float vertices[] = { + 0.0f, 2.0f, + 0.0f, 0.0f, + 2.0f, 0.0f + }; + vbo = generate_vbo(2, GL_FLOAT, sizeof(vertices), vertices); + check_error(); + + // v210 compute shader. + if (v210Converter::has_hardware_support()) { + string v210_shader_src = R"(#version 150 +#extension GL_ARB_compute_shader : enable +#extension GL_ARB_shader_image_load_store : enable +layout(local_size_x=2, local_size_y=16) in; +layout(r16) uniform restrict readonly image2D in_y; +uniform sampler2D in_cbcr; // Of type RG16. +layout(rgb10_a2) uniform restrict writeonly image2D outbuf; +uniform float inv_width, inv_height; + +void main() +{ + int xb = int(gl_GlobalInvocationID.x); // X block number. + int y = int(gl_GlobalInvocationID.y); // Y (actual line). + float yf = (gl_GlobalInvocationID.y + 0.5f) * inv_height; // Y float coordinate. + + // Load and scale CbCr values, sampling in-between the texels to get + // to (left/4 + center/2 + right/4). + vec2 pix_cbcr[3]; + for (int i = 0; i < 3; ++i) { + vec2 a = texture(in_cbcr, vec2((xb * 6 + i * 2) * inv_width, yf)).xy; + vec2 b = texture(in_cbcr, vec2((xb * 6 + i * 2 + 1) * inv_width, yf)).xy; + pix_cbcr[i] = (a + b) * (0.5 * 65535.0 / 1023.0); + } + + // Load and scale the Y values. Note that we use integer coordinates here, + // so we don't need to offset by 0.5. + float pix_y[6]; + for (int i = 0; i < 6; ++i) { + pix_y[i] = imageLoad(in_y, ivec2(xb * 6 + i, y)).x * (65535.0 / 1023.0); + } + + imageStore(outbuf, ivec2(xb * 4 + 0, y), vec4(pix_cbcr[0].x, pix_y[0], pix_cbcr[0].y, 1.0)); + imageStore(outbuf, ivec2(xb * 4 + 1, y), vec4(pix_y[1], pix_cbcr[1].x, pix_y[2], 1.0)); + imageStore(outbuf, ivec2(xb * 4 + 2, y), vec4(pix_cbcr[1].y, pix_y[3], pix_cbcr[2].x, 1.0)); + imageStore(outbuf, ivec2(xb * 4 + 3, y), vec4(pix_y[4], pix_cbcr[2].y, pix_y[5], 1.0)); +} +)"; + GLuint shader_num = movit::compile_shader(v210_shader_src, GL_COMPUTE_SHADER); + check_error(); + v210_program_num = glCreateProgram(); + check_error(); + glAttachShader(v210_program_num, shader_num); + check_error(); + glLinkProgram(v210_program_num); + check_error(); + + GLint success; + glGetProgramiv(v210_program_num, GL_LINK_STATUS, &success); + check_error(); + if (success == GL_FALSE) { + GLchar error_log[1024] = {0}; + glGetProgramInfoLog(v210_program_num, 1024, nullptr, error_log); + fprintf(stderr, "Error linking program: %s\n", error_log); + exit(1); + } + + v210_in_y_pos = glGetUniformLocation(v210_program_num, "in_y"); + check_error(); + v210_in_cbcr_pos = glGetUniformLocation(v210_program_num, "in_cbcr"); + check_error(); + v210_outbuf_pos = glGetUniformLocation(v210_program_num, "outbuf"); + check_error(); + v210_inv_width_pos = glGetUniformLocation(v210_program_num, "inv_width"); + check_error(); + v210_inv_height_pos = glGetUniformLocation(v210_program_num, "inv_height"); + check_error(); + } else { + v210_program_num = 0; + } +} + +ChromaSubsampler::~ChromaSubsampler() +{ + resource_pool->release_glsl_program(cbcr_program_num); + check_error(); + resource_pool->release_glsl_program(uyvy_program_num); + check_error(); + glDeleteBuffers(1, &vbo); + check_error(); + if (v210_program_num != 0) { + glDeleteProgram(v210_program_num); + check_error(); + } +} + +void ChromaSubsampler::subsample_chroma(GLuint cbcr_tex, unsigned width, unsigned height, GLuint dst_tex, GLuint dst2_tex) +{ + GLuint vao = resource_pool->create_vec2_vao({ cbcr_position_attribute_index, cbcr_texcoord_attribute_index }, vbo); + glBindVertexArray(vao); + check_error(); + + // Extract Cb/Cr. + GLuint fbo; + if (dst2_tex <= 0) { + fbo = resource_pool->create_fbo(dst_tex); + } else { + fbo = resource_pool->create_fbo(dst_tex, dst2_tex); + } + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + glViewport(0, 0, width/2, height/2); + check_error(); + + glUseProgram(cbcr_program_num); + check_error(); + + glActiveTexture(GL_TEXTURE0); + check_error(); + glBindTexture(GL_TEXTURE_2D, cbcr_tex); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + + glUniform2f(cbcr_chroma_offset_0_location, -1.0f / width, 0.0f); + check_error(); + glUniform2f(cbcr_chroma_offset_1_location, -0.0f / width, 0.0f); + check_error(); + glUniform1i(cbcr_texture_sampler_uniform, 0); + + glDrawArrays(GL_TRIANGLES, 0, 3); + check_error(); + + glUseProgram(0); + check_error(); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + check_error(); + glBindVertexArray(0); + check_error(); + + resource_pool->release_fbo(fbo); + resource_pool->release_vec2_vao(vao); +} + +void ChromaSubsampler::create_uyvy(GLuint y_tex, GLuint cbcr_tex, unsigned width, unsigned height, GLuint dst_tex) +{ + GLuint vao = resource_pool->create_vec2_vao({ cbcr_position_attribute_index, cbcr_texcoord_attribute_index }, vbo); + glBindVertexArray(vao); + check_error(); + + glBindVertexArray(vao); + check_error(); + + GLuint fbo = resource_pool->create_fbo(dst_tex); + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + glViewport(0, 0, width/2, height); + check_error(); + + glUseProgram(uyvy_program_num); + check_error(); + + glUniform1i(uyvy_y_texture_sampler_uniform, 0); + check_error(); + glUniform1i(uyvy_cbcr_texture_sampler_uniform, 1); + check_error(); + + glActiveTexture(GL_TEXTURE0); + check_error(); + glBindTexture(GL_TEXTURE_2D, y_tex); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + + glActiveTexture(GL_TEXTURE1); + check_error(); + glBindTexture(GL_TEXTURE_2D, cbcr_tex); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + + glUniform2f(uyvy_luma_offset_0_location, -0.5f / width, 0.0f); + check_error(); + glUniform2f(uyvy_luma_offset_1_location, 0.5f / width, 0.0f); + check_error(); + glUniform2f(uyvy_chroma_offset_0_location, -1.0f / width, 0.0f); + check_error(); + glUniform2f(uyvy_chroma_offset_1_location, -0.0f / width, 0.0f); + check_error(); + + glBindBuffer(GL_ARRAY_BUFFER, vbo); + check_error(); + + glDrawArrays(GL_TRIANGLES, 0, 3); + check_error(); + + glActiveTexture(GL_TEXTURE0); + check_error(); + glUseProgram(0); + check_error(); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + check_error(); + glBindVertexArray(0); + check_error(); + + resource_pool->release_fbo(fbo); + resource_pool->release_vec2_vao(vao); +} + +void ChromaSubsampler::create_v210(GLuint y_tex, GLuint cbcr_tex, unsigned width, unsigned height, GLuint dst_tex) +{ + assert(v210_program_num != 0); + + glUseProgram(v210_program_num); + check_error(); + + glUniform1i(v210_in_y_pos, 0); + check_error(); + glUniform1i(v210_in_cbcr_pos, 1); + check_error(); + glUniform1i(v210_outbuf_pos, 2); + check_error(); + glUniform1f(v210_inv_width_pos, 1.0 / width); + check_error(); + glUniform1f(v210_inv_height_pos, 1.0 / height); + check_error(); + + glActiveTexture(GL_TEXTURE0); + check_error(); + glBindTexture(GL_TEXTURE_2D, y_tex); // We don't actually need to bind it, but we need to set the state. + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + glBindImageTexture(0, y_tex, 0, GL_FALSE, 0, GL_READ_ONLY, GL_R16); // This is the real bind. + check_error(); + + glActiveTexture(GL_TEXTURE1); + check_error(); + glBindTexture(GL_TEXTURE_2D, cbcr_tex); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + + glBindImageTexture(2, dst_tex, 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGB10_A2); + check_error(); + + // Actually run the shader. We use workgroups of size 2x16 threadst , and each thread + // processes 6x1 input pixels, so round up to number of 12x16 pixel blocks. + glDispatchCompute((width + 11) / 12, (height + 15) / 16, 1); + + glBindTexture(GL_TEXTURE_2D, 0); + check_error(); + glActiveTexture(GL_TEXTURE0); + check_error(); + glUseProgram(0); + check_error(); +} diff --git a/nageru/chroma_subsampler.h b/nageru/chroma_subsampler.h new file mode 100644 index 0000000..8e9ff4e --- /dev/null +++ b/nageru/chroma_subsampler.h @@ -0,0 +1,59 @@ +#ifndef _CHROMA_SUBSAMPLER_H +#define _CHROMA_SUBSAMPLER_H 1 + +#include + +namespace movit { + +class ResourcePool; + +} // namespace movit + +class ChromaSubsampler { +public: + ChromaSubsampler(movit::ResourcePool *resource_pool); + ~ChromaSubsampler(); + + // Subsamples chroma (packed Cb and Cr) 2x2 to yield chroma suitable for + // NV12 (semiplanar 4:2:0). Chroma positioning is left/center (H.264 convention). + // width and height are the dimensions (in pixels) of the input texture. + // + // You can get two equal copies if you'd like; just set dst2_tex to a texture + // number and it will receive an exact copy of what goes into dst_tex. + void subsample_chroma(GLuint cbcr_tex, unsigned width, unsigned height, GLuint dst_tex, GLuint dst2_tex = 0); + + // Subsamples and interleaves luma and chroma to give 4:2:2 packed Y'CbCr (UYVY). + // Chroma positioning is left (H.264 convention). + // width and height are the dimensions (in pixels) of the input textures. + void create_uyvy(GLuint y_tex, GLuint cbcr_tex, unsigned width, unsigned height, GLuint dst_tex); + + // Subsamples and interleaves luma and chroma to give 10-bit 4:2:2 + // packed Y'CbCr (v210); see v210converter.h for more information on + // the format. Luma and chroma are assumed to be 10-bit data packed + // into 16-bit textures. Chroma positioning is left (H.264 convention). + // width and height are the dimensions (in pixels) of the input textures; + // Requires compute shaders; check v210Converter::has_hardware_support(). + void create_v210(GLuint y_tex, GLuint cbcr_tex, unsigned width, unsigned height, GLuint dst_tex); + +private: + movit::ResourcePool *resource_pool; + + GLuint vbo; // Holds position and texcoord data. + + GLuint cbcr_program_num; // Owned by . + GLuint cbcr_texture_sampler_uniform; + GLint cbcr_position_attribute_index, cbcr_texcoord_attribute_index; + GLuint cbcr_chroma_offset_0_location, cbcr_chroma_offset_1_location; + + GLuint uyvy_program_num; // Owned by . + GLuint uyvy_y_texture_sampler_uniform, uyvy_cbcr_texture_sampler_uniform; + GLint uyvy_position_attribute_index, uyvy_texcoord_attribute_index; + GLuint uyvy_luma_offset_0_location, uyvy_luma_offset_1_location; + GLuint uyvy_chroma_offset_0_location, uyvy_chroma_offset_1_location; + + GLuint v210_program_num; // Compute shader, so owned by ourselves. Can be 0. + GLuint v210_in_y_pos, v210_in_cbcr_pos, v210_outbuf_pos; + GLuint v210_inv_width_pos, v210_inv_height_pos; +}; + +#endif // !defined(_CHROMA_SUBSAMPLER_H) diff --git a/nageru/clickable_label.h b/nageru/clickable_label.h new file mode 100644 index 0000000..cc82168 --- /dev/null +++ b/nageru/clickable_label.h @@ -0,0 +1,26 @@ +#ifndef _CLICKABLE_LABEL_H +#define _CLICKABLE_LABEL_H 1 + +// Just like a normal QLabel, except that it can also emit a clicked signal. + +#include + +class QMouseEvent; + +class ClickableLabel : public QLabel { + Q_OBJECT + +public: + ClickableLabel(QWidget *parent) : QLabel(parent) {} + +signals: + void clicked(); + +protected: + void mousePressEvent(QMouseEvent *event) override + { + emit clicked(); + } +}; + +#endif // !defined(_CLICKABLE_LABEL_H) diff --git a/nageru/compression_reduction_meter.cpp b/nageru/compression_reduction_meter.cpp new file mode 100644 index 0000000..a59a71e --- /dev/null +++ b/nageru/compression_reduction_meter.cpp @@ -0,0 +1,100 @@ +#include "compression_reduction_meter.h" + +#include +#include +#include "piecewise_interpolator.h" +#include "vu_common.h" + +class QPaintEvent; +class QResizeEvent; + +using namespace std; + +namespace { + +vector control_points = { + { 60.0f, 6.0f }, + { 30.0f, 5.0f }, + { 18.0f, 4.0f }, + { 12.0f, 3.0f }, + { 6.0f, 2.0f }, + { 3.0f, 1.0f }, + { 0.0f, 0.0f } +}; +PiecewiseInterpolator interpolator(control_points); + +} // namespace + +CompressionReductionMeter::CompressionReductionMeter(QWidget *parent) + : QWidget(parent) +{ +} + +void CompressionReductionMeter::resizeEvent(QResizeEvent *event) +{ + recalculate_pixmaps(); +} + +void CompressionReductionMeter::paintEvent(QPaintEvent *event) +{ + QPainter painter(this); + + float level_db; + { + unique_lock lock(level_mutex); + level_db = this->level_db; + } + + int on_pos = lrint(db_to_pos(level_db)); + + QRect on_rect(0, 0, width(), on_pos); + QRect off_rect(0, on_pos, width(), height()); + + painter.drawPixmap(on_rect, on_pixmap, on_rect); + painter.drawPixmap(off_rect, off_pixmap, off_rect); +} + +void CompressionReductionMeter::recalculate_pixmaps() +{ + constexpr int y_offset = text_box_height / 2; + constexpr int text_margin = 5; + float margin = 0.5 * (width() - meter_width); + + on_pixmap = QPixmap(width(), height()); + QPainter on_painter(&on_pixmap); + on_painter.fillRect(0, 0, width(), height(), parentWidget()->palette().window()); + draw_vu_meter(on_painter, width(), meter_height(), margin, 2.0, true, min_level, max_level, /*flip=*/true, y_offset); + draw_scale(&on_painter, 0.5 * width() + 0.5 * meter_width + text_margin); + + off_pixmap = QPixmap(width(), height()); + QPainter off_painter(&off_pixmap); + off_painter.fillRect(0, 0, width(), height(), parentWidget()->palette().window()); + draw_vu_meter(off_painter, width(), meter_height(), margin, 2.0, false, min_level, max_level, /*flip=*/true, y_offset); + draw_scale(&off_painter, 0.5 * width() + 0.5 * meter_width + text_margin); +} + +void CompressionReductionMeter::draw_scale(QPainter *painter, int x_pos) +{ + QFont font; + font.setPointSize(8); + painter->setPen(Qt::black); + painter->setFont(font); + for (size_t i = 0; i < control_points.size(); ++i) { + char buf[256]; + snprintf(buf, 256, "%.0f", control_points[i].db_value); + double y = db_to_pos(control_points[i].db_value); + painter->drawText(QRect(x_pos, y - text_box_height / 2, text_box_width, text_box_height), + Qt::AlignCenter | Qt::AlignVCenter, buf); + } +} + +double CompressionReductionMeter::db_to_pos(double level_db) const +{ + float value = interpolator.db_to_fraction(level_db); + return height() - lufs_to_pos(value, meter_height(), min_level, max_level) - text_box_height / 2; +} + +int CompressionReductionMeter::meter_height() const +{ + return height() - text_box_height; +} diff --git a/nageru/compression_reduction_meter.h b/nageru/compression_reduction_meter.h new file mode 100644 index 0000000..5890c13 --- /dev/null +++ b/nageru/compression_reduction_meter.h @@ -0,0 +1,54 @@ +#ifndef COMPRESSION_REDUCTION_METER_H +#define COMPRESSION_REDUCTION_METER_H + +// A meter that goes downwards instead of upwards, and has a non-linear scale. + +#include +#include +#include +#include +#include + +#include "piecewise_interpolator.h" + +class QObject; +class QPaintEvent; +class QResizeEvent; + +class CompressionReductionMeter : public QWidget +{ + Q_OBJECT + +public: + CompressionReductionMeter(QWidget *parent); + + void set_reduction_db(float level_db) { + std::unique_lock lock(level_mutex); + this->level_db = level_db; + QMetaObject::invokeMethod(this, "update", Qt::AutoConnection); + } + +private: + void resizeEvent(QResizeEvent *event) override; + void paintEvent(QPaintEvent *event) override; + void recalculate_pixmaps(); + void draw_scale(QPainter *painter, int x_pos); + double db_to_pos(double db) const; + int meter_height() const; + + std::mutex level_mutex; + float level_db = 0.0f; + + static constexpr float min_level = 0.0f; // Must match control_points (in the .cpp file). + static constexpr float max_level = 6.0f; // Same. + static constexpr int meter_width = 20; + + // Size of the text box. The meter will be shrunk to make room for the text box + // (half the height) on both sides. + static constexpr int text_box_width = 15; + static constexpr int text_box_height = 10; + + QPixmap on_pixmap, off_pixmap; +}; + +#endif diff --git a/nageru/context.cpp b/nageru/context.cpp new file mode 100644 index 0000000..eb62183 --- /dev/null +++ b/nageru/context.cpp @@ -0,0 +1,50 @@ +#include + +#include + +#include +#include +#include +#include +#include + +QGLWidget *global_share_widget = nullptr; +bool using_egl = false; + +using namespace std; + +QSurface *create_surface(const QSurfaceFormat &format) +{ + QOffscreenSurface *surface = new QOffscreenSurface; + surface->setFormat(format); + surface->create(); + if (!surface->isValid()) { + fprintf(stderr, "ERROR: surface not valid!\n"); + exit(1); + } + return surface; +} + +QSurface *create_surface_with_same_format(const QSurface *surface) +{ + return create_surface(surface->format()); +} + +QOpenGLContext *create_context(const QSurface *surface) +{ + QOpenGLContext *context = new QOpenGLContext; + context->setShareContext(global_share_widget->context()->contextHandle()); + context->setFormat(surface->format()); + context->create(); + return context; +} + +bool make_current(QOpenGLContext *context, QSurface *surface) +{ + return context->makeCurrent(surface); +} + +void delete_context(QOpenGLContext *context) +{ + delete context; +} diff --git a/nageru/context.h b/nageru/context.h new file mode 100644 index 0000000..13dbf24 --- /dev/null +++ b/nageru/context.h @@ -0,0 +1,16 @@ + +// Needs to be in its own file because Qt and libepoxy seemingly don't coexist well +// within the same file. + +class QSurface; +class QOpenGLContext; +class QSurfaceFormat; +class QGLWidget; + +extern bool using_egl; +extern QGLWidget *global_share_widget; +QSurface *create_surface(const QSurfaceFormat &format); +QSurface *create_surface_with_same_format(const QSurface *surface); +QOpenGLContext *create_context(const QSurface *surface); +bool make_current(QOpenGLContext *context, QSurface *surface); +void delete_context(QOpenGLContext *context); diff --git a/nageru/context_menus.cpp b/nageru/context_menus.cpp new file mode 100644 index 0000000..790de94 --- /dev/null +++ b/nageru/context_menus.cpp @@ -0,0 +1,66 @@ +#include +#include +#include + +#include + +#include "mixer.h" + +using namespace std; + +void fill_hdmi_sdi_output_device_menu(QMenu *menu) +{ + menu->clear(); + QActionGroup *card_group = new QActionGroup(menu); + int current_card = global_mixer->get_output_card_index(); + + QAction *none_action = new QAction("None", card_group); + none_action->setCheckable(true); + if (current_card == -1) { + none_action->setChecked(true); + } + QObject::connect(none_action, &QAction::triggered, []{ global_mixer->set_output_card(-1); }); + menu->addAction(none_action); + + unsigned num_cards = global_mixer->get_num_cards(); + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + if (!global_mixer->card_can_be_used_as_output(card_index)) { + continue; + } + + QString description(QString::fromStdString(global_mixer->get_output_card_description(card_index))); + QAction *action = new QAction(description, card_group); + action->setCheckable(true); + if (current_card == int(card_index)) { + action->setChecked(true); + } + QObject::connect(action, &QAction::triggered, [card_index]{ global_mixer->set_output_card(card_index); }); + menu->addAction(action); + } +} + +void fill_hdmi_sdi_output_resolution_menu(QMenu *menu) +{ + menu->clear(); + int current_card = global_mixer->get_output_card_index(); + if (current_card == -1) { + menu->setEnabled(false); + return; + } + + menu->setEnabled(true); + QActionGroup *resolution_group = new QActionGroup(menu); + uint32_t current_mode = global_mixer->get_output_video_mode(); + map video_modes = global_mixer->get_available_output_video_modes(); + for (const auto &mode : video_modes) { + QString description(QString::fromStdString(mode.second.name)); + QAction *action = new QAction(description, resolution_group); + action->setCheckable(true); + if (current_mode == mode.first) { + action->setChecked(true); + } + const uint32_t mode_id = mode.first; + QObject::connect(action, &QAction::triggered, [mode_id]{ global_mixer->set_output_video_mode(mode_id); }); + menu->addAction(action); + } +} diff --git a/nageru/context_menus.h b/nageru/context_menus.h new file mode 100644 index 0000000..2f9005d --- /dev/null +++ b/nageru/context_menus.h @@ -0,0 +1,19 @@ +#ifndef _CONTEXT_MENUS_H +#define _CONTEXT_MENUS_H 1 + +// Some context menus for controlling various I/O selections, +// based on data from Mixer. + +class QMenu; + +// Populate a submenu for selecting output card, with an action for each card. +// Will call into the mixer on trigger. +void fill_hdmi_sdi_output_device_menu(QMenu *menu); + +// Populate a submenu for choosing the output resolution. Since this is +// card-dependent, the entire menu is disabled if we haven't chosen a card +// (but it's still there so that the user will know it exists). +// Will call into the mixer on trigger. +void fill_hdmi_sdi_output_resolution_menu(QMenu *menu); + +#endif // !defined(_CONTEXT_MENUS_H) diff --git a/nageru/correlation_measurer.cpp b/nageru/correlation_measurer.cpp new file mode 100644 index 0000000..d0150a0 --- /dev/null +++ b/nageru/correlation_measurer.cpp @@ -0,0 +1,72 @@ +// Adapted from Adriaensen's project Zita-mu1 (as of January 2016). +// Original copyright follows: +// +// Copyright (C) 2008-2015 Fons Adriaensen +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include "correlation_measurer.h" + +#include +#include +#include + +using namespace std; + +CorrelationMeasurer::CorrelationMeasurer(unsigned sample_rate, + float lowpass_cutoff_hz, + float falloff_seconds) + : w1(2.0 * M_PI * lowpass_cutoff_hz / sample_rate), + w2(1.0 / (falloff_seconds * sample_rate)) +{ +} + +void CorrelationMeasurer::reset() +{ + zl = zr = zll = zlr = zrr = 0.0f; +} + +void CorrelationMeasurer::process_samples(const std::vector &samples) +{ + assert(samples.size() % 2 == 0); + + // The compiler isn't always happy about modifying members, + // since it doesn't always know they can't alias on . + // Help it out a bit. + float l = zl, r = zr, ll = zll, lr = zlr, rr = zrr; + const float w1c = w1, w2c = w2; + + for (size_t i = 0; i < samples.size(); i += 2) { + // The 1e-15f epsilon is to avoid denormals. + // TODO: Just set the SSE flush-to-zero flags instead. + l += w1c * (samples[i + 0] - l) + 1e-15f; + r += w1c * (samples[i + 1] - r) + 1e-15f; + lr += w2c * (l * r - lr); + ll += w2c * (l * l - ll); + rr += w2c * (r * r - rr); + } + + zl = l; + zr = r; + zll = ll; + zlr = lr; + zrr = rr; +} + +float CorrelationMeasurer::get_correlation() const +{ + // The 1e-12f epsilon is to avoid division by zero. + // zll and zrr are both always non-negative, so we do not risk negative values. + return zlr / sqrt(zll * zrr + 1e-12f); +} diff --git a/nageru/correlation_measurer.h b/nageru/correlation_measurer.h new file mode 100644 index 0000000..0c0ac72 --- /dev/null +++ b/nageru/correlation_measurer.h @@ -0,0 +1,56 @@ +#ifndef _CORRELATION_MEASURER_H +#define _CORRELATION_MEASURER_H 1 + +// Measurement of left/right stereo correlation. +1 is pure mono +// (okay but not ideal), 0 is no correlation (usually bad, unless +// it is due to silence), strongly negative values means inverted +// phase (bad). Typical values for e.g. music would be somewhere +// around +0.7, although you can expect it to vary a bit. +// +// This is, of course, based on the regular Pearson correlation, +// where µ_L and µ_R is taken to be 0 (ie., no DC offset). It is +// filtered through a simple IIR filter so that older values are +// weighed less than newer, depending on . +// +// +// Adapted from Adriaensen's project Zita-mu1 (as of January 2016). +// Original copyright follows: +// +// Copyright (C) 2008-2015 Fons Adriaensen +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include + +class CorrelationMeasurer { +public: + CorrelationMeasurer(unsigned sample_rate, float lowpass_cutoff_hz = 1000.0f, + float falloff_seconds = 0.150f); + void process_samples(const std::vector &samples); // Taken to be stereo, interleaved. + void reset(); + float get_correlation() const; + +private: + float w1, w2; + + // Filtered values of left and right channel, respectively. + float zl = 0.0f, zr = 0.0f; + + // Filtered values of l², r² and lr (where l and r are the filtered + // versions, given by zl and zr). Together, they make up what we need + // to calculate the correlation. + float zll = 0.0f, zlr = 0.0f, zrr = 0.0f; +}; + +#endif // !defined(_CORRELATION_MEASURER_H) diff --git a/nageru/correlation_meter.cpp b/nageru/correlation_meter.cpp new file mode 100644 index 0000000..7b7683a --- /dev/null +++ b/nageru/correlation_meter.cpp @@ -0,0 +1,63 @@ +#include "correlation_meter.h" + +#include +#include + +#include +#include +#include +#include + +class QPaintEvent; +class QResizeEvent; + +using namespace std; + +CorrelationMeter::CorrelationMeter(QWidget *parent) + : QWidget(parent) +{ +} + +void CorrelationMeter::resizeEvent(QResizeEvent *event) +{ + on_pixmap = QPixmap(width(), height()); + QPainter on_painter(&on_pixmap); + QLinearGradient on(0, 0, width(), 0); + on.setColorAt(0.0f, QColor(255, 0, 0)); + on.setColorAt(0.5f, QColor(255, 255, 0)); + on.setColorAt(0.8f, QColor(0, 255, 0)); + on.setColorAt(0.95f, QColor(255, 255, 0)); + on_painter.fillRect(0, 0, width(), height(), Qt::black); + on_painter.fillRect(1, 1, width() - 2, height() - 2, on); + + off_pixmap = QPixmap(width(), height()); + QPainter off_painter(&off_pixmap); + QLinearGradient off(0, 0, width(), 0); + off.setColorAt(0.0f, QColor(127, 0, 0)); + off.setColorAt(0.5f, QColor(127, 127, 0)); + off.setColorAt(0.8f, QColor(0, 127, 0)); + off.setColorAt(0.95f, QColor(127, 127, 0)); + off_painter.fillRect(0, 0, width(), height(), Qt::black); + off_painter.fillRect(1, 1, width() - 2, height() - 2, off); +} + +void CorrelationMeter::paintEvent(QPaintEvent *event) +{ + QPainter painter(this); + + float correlation; + { + unique_lock lock(correlation_mutex); + correlation = this->correlation; + } + + // Just in case. + correlation = std::min(std::max(correlation, -1.0f), 1.0f); + + int pos = 3 + lrintf(0.5f * (correlation + 1.0f) * (width() - 6)); + QRect off_rect(0, 0, width(), height()); + QRect on_rect(pos - 2, 0, 5, height()); + + painter.drawPixmap(off_rect, off_pixmap, off_rect); + painter.drawPixmap(on_rect, on_pixmap, on_rect); +} diff --git a/nageru/correlation_meter.h b/nageru/correlation_meter.h new file mode 100644 index 0000000..ea01e04 --- /dev/null +++ b/nageru/correlation_meter.h @@ -0,0 +1,37 @@ +#ifndef CORRELATION_METER_H +#define CORRELATION_METER_H + +#include + +#include +#include +#include + +class QObject; +class QPaintEvent; +class QResizeEvent; + +class CorrelationMeter : public QWidget +{ + Q_OBJECT + +public: + CorrelationMeter(QWidget *parent); + + void set_correlation(float correlation) { + std::unique_lock lock(correlation_mutex); + this->correlation = correlation; + QMetaObject::invokeMethod(this, "update", Qt::AutoConnection); + } + +private: + void resizeEvent(QResizeEvent *event) override; + void paintEvent(QPaintEvent *event) override; + + std::mutex correlation_mutex; + float correlation = 0.0f; + + QPixmap on_pixmap, off_pixmap; +}; + +#endif diff --git a/nageru/db.h b/nageru/db.h new file mode 100644 index 0000000..53261ab --- /dev/null +++ b/nageru/db.h @@ -0,0 +1,11 @@ +#ifndef _DB_H +#define _DB_H 1 + +// Utility routines for working with decibels. + +#include + +static inline double from_db(double db) { return pow(10.0, db / 20.0); } +static inline double to_db(double val) { return 20.0 * log10(val); } + +#endif // !defined(_DB_H) diff --git a/nageru/decklink/DeckLinkAPI.h b/nageru/decklink/DeckLinkAPI.h new file mode 100755 index 0000000..2a0f90a --- /dev/null +++ b/nageru/decklink/DeckLinkAPI.h @@ -0,0 +1,946 @@ +/* -LICENSE-START- +** Copyright (c) 2015 Blackmagic Design +** +** Permission is hereby granted, free of charge, to any person or organization +** obtaining a copy of the software and accompanying documentation covered by +** this license (the "Software") to use, reproduce, display, distribute, +** execute, and transmit the Software, and to prepare derivative works of the +** Software, and to permit third-parties to whom the Software is furnished to +** do so, all subject to the following: +** +** The copyright notices in the Software and this entire statement, including +** the above license grant, this restriction and the following disclaimer, +** must be included in all copies of the Software, in whole or in part, and +** all derivative works of the Software, unless such copies or derivative +** works are solely in the form of machine-executable object code generated by +** a source language processor. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +** -LICENSE-END- +*/ + +#ifndef BMD_DECKLINKAPI_H +#define BMD_DECKLINKAPI_H + + +#ifndef BMD_CONST + #if defined(_MSC_VER) + #define BMD_CONST __declspec(selectany) static const + #else + #define BMD_CONST static const + #endif +#endif + +/* DeckLink API */ + +#include +#include "LinuxCOM.h" + +#include "DeckLinkAPITypes.h" +#include "DeckLinkAPIModes.h" +#include "DeckLinkAPIDiscovery.h" +#include "DeckLinkAPIConfiguration.h" +#include "DeckLinkAPIDeckControl.h" + +#define BLACKMAGIC_DECKLINK_API_MAGIC 1 + +// Type Declarations + + +// Interface ID Declarations + +BMD_CONST REFIID IID_IDeckLinkVideoOutputCallback = /* 20AA5225-1958-47CB-820B-80A8D521A6EE */ {0x20,0xAA,0x52,0x25,0x19,0x58,0x47,0xCB,0x82,0x0B,0x80,0xA8,0xD5,0x21,0xA6,0xEE}; +BMD_CONST REFIID IID_IDeckLinkInputCallback = /* DD04E5EC-7415-42AB-AE4A-E80C4DFC044A */ {0xDD,0x04,0xE5,0xEC,0x74,0x15,0x42,0xAB,0xAE,0x4A,0xE8,0x0C,0x4D,0xFC,0x04,0x4A}; +BMD_CONST REFIID IID_IDeckLinkEncoderInputCallback = /* ACF13E61-F4A0-4974-A6A7-59AFF6268B31 */ {0xAC,0xF1,0x3E,0x61,0xF4,0xA0,0x49,0x74,0xA6,0xA7,0x59,0xAF,0xF6,0x26,0x8B,0x31}; +BMD_CONST REFIID IID_IDeckLinkMemoryAllocator = /* B36EB6E7-9D29-4AA8-92EF-843B87A289E8 */ {0xB3,0x6E,0xB6,0xE7,0x9D,0x29,0x4A,0xA8,0x92,0xEF,0x84,0x3B,0x87,0xA2,0x89,0xE8}; +BMD_CONST REFIID IID_IDeckLinkAudioOutputCallback = /* 403C681B-7F46-4A12-B993-2BB127084EE6 */ {0x40,0x3C,0x68,0x1B,0x7F,0x46,0x4A,0x12,0xB9,0x93,0x2B,0xB1,0x27,0x08,0x4E,0xE6}; +BMD_CONST REFIID IID_IDeckLinkIterator = /* 50FB36CD-3063-4B73-BDBB-958087F2D8BA */ {0x50,0xFB,0x36,0xCD,0x30,0x63,0x4B,0x73,0xBD,0xBB,0x95,0x80,0x87,0xF2,0xD8,0xBA}; +BMD_CONST REFIID IID_IDeckLinkAPIInformation = /* 7BEA3C68-730D-4322-AF34-8A7152B532A4 */ {0x7B,0xEA,0x3C,0x68,0x73,0x0D,0x43,0x22,0xAF,0x34,0x8A,0x71,0x52,0xB5,0x32,0xA4}; +BMD_CONST REFIID IID_IDeckLinkOutput = /* CC5C8A6E-3F2F-4B3A-87EA-FD78AF300564 */ {0xCC,0x5C,0x8A,0x6E,0x3F,0x2F,0x4B,0x3A,0x87,0xEA,0xFD,0x78,0xAF,0x30,0x05,0x64}; +BMD_CONST REFIID IID_IDeckLinkInput = /* AF22762B-DFAC-4846-AA79-FA8883560995 */ {0xAF,0x22,0x76,0x2B,0xDF,0xAC,0x48,0x46,0xAA,0x79,0xFA,0x88,0x83,0x56,0x09,0x95}; +BMD_CONST REFIID IID_IDeckLinkEncoderInput = /* 270587DA-6B7D-42E7-A1F0-6D853F581185 */ {0x27,0x05,0x87,0xDA,0x6B,0x7D,0x42,0xE7,0xA1,0xF0,0x6D,0x85,0x3F,0x58,0x11,0x85}; +BMD_CONST REFIID IID_IDeckLinkVideoFrame = /* 3F716FE0-F023-4111-BE5D-EF4414C05B17 */ {0x3F,0x71,0x6F,0xE0,0xF0,0x23,0x41,0x11,0xBE,0x5D,0xEF,0x44,0x14,0xC0,0x5B,0x17}; +BMD_CONST REFIID IID_IDeckLinkMutableVideoFrame = /* 69E2639F-40DA-4E19-B6F2-20ACE815C390 */ {0x69,0xE2,0x63,0x9F,0x40,0xDA,0x4E,0x19,0xB6,0xF2,0x20,0xAC,0xE8,0x15,0xC3,0x90}; +BMD_CONST REFIID IID_IDeckLinkVideoFrame3DExtensions = /* DA0F7E4A-EDC7-48A8-9CDD-2DB51C729CD7 */ {0xDA,0x0F,0x7E,0x4A,0xED,0xC7,0x48,0xA8,0x9C,0xDD,0x2D,0xB5,0x1C,0x72,0x9C,0xD7}; +BMD_CONST REFIID IID_IDeckLinkVideoInputFrame = /* 05CFE374-537C-4094-9A57-680525118F44 */ {0x05,0xCF,0xE3,0x74,0x53,0x7C,0x40,0x94,0x9A,0x57,0x68,0x05,0x25,0x11,0x8F,0x44}; +BMD_CONST REFIID IID_IDeckLinkVideoFrameAncillary = /* 732E723C-D1A4-4E29-9E8E-4A88797A0004 */ {0x73,0x2E,0x72,0x3C,0xD1,0xA4,0x4E,0x29,0x9E,0x8E,0x4A,0x88,0x79,0x7A,0x00,0x04}; +BMD_CONST REFIID IID_IDeckLinkEncoderPacket = /* B693F36C-316E-4AF1-B6C2-F389A4BCA620 */ {0xB6,0x93,0xF3,0x6C,0x31,0x6E,0x4A,0xF1,0xB6,0xC2,0xF3,0x89,0xA4,0xBC,0xA6,0x20}; +BMD_CONST REFIID IID_IDeckLinkEncoderVideoPacket = /* 4E7FD944-E8C7-4EAC-B8C0-7B77F80F5AE0 */ {0x4E,0x7F,0xD9,0x44,0xE8,0xC7,0x4E,0xAC,0xB8,0xC0,0x7B,0x77,0xF8,0x0F,0x5A,0xE0}; +BMD_CONST REFIID IID_IDeckLinkEncoderAudioPacket = /* 49E8EDC8-693B-4E14-8EF6-12C658F5A07A */ {0x49,0xE8,0xED,0xC8,0x69,0x3B,0x4E,0x14,0x8E,0xF6,0x12,0xC6,0x58,0xF5,0xA0,0x7A}; +BMD_CONST REFIID IID_IDeckLinkH265NALPacket = /* 639C8E0B-68D5-4BDE-A6D4-95F3AEAFF2E7 */ {0x63,0x9C,0x8E,0x0B,0x68,0xD5,0x4B,0xDE,0xA6,0xD4,0x95,0xF3,0xAE,0xAF,0xF2,0xE7}; +BMD_CONST REFIID IID_IDeckLinkAudioInputPacket = /* E43D5870-2894-11DE-8C30-0800200C9A66 */ {0xE4,0x3D,0x58,0x70,0x28,0x94,0x11,0xDE,0x8C,0x30,0x08,0x00,0x20,0x0C,0x9A,0x66}; +BMD_CONST REFIID IID_IDeckLinkScreenPreviewCallback = /* B1D3F49A-85FE-4C5D-95C8-0B5D5DCCD438 */ {0xB1,0xD3,0xF4,0x9A,0x85,0xFE,0x4C,0x5D,0x95,0xC8,0x0B,0x5D,0x5D,0xCC,0xD4,0x38}; +BMD_CONST REFIID IID_IDeckLinkGLScreenPreviewHelper = /* 504E2209-CAC7-4C1A-9FB4-C5BB6274D22F */ {0x50,0x4E,0x22,0x09,0xCA,0xC7,0x4C,0x1A,0x9F,0xB4,0xC5,0xBB,0x62,0x74,0xD2,0x2F}; +BMD_CONST REFIID IID_IDeckLinkNotificationCallback = /* B002A1EC-070D-4288-8289-BD5D36E5FF0D */ {0xB0,0x02,0xA1,0xEC,0x07,0x0D,0x42,0x88,0x82,0x89,0xBD,0x5D,0x36,0xE5,0xFF,0x0D}; +BMD_CONST REFIID IID_IDeckLinkNotification = /* 0A1FB207-E215-441B-9B19-6FA1575946C5 */ {0x0A,0x1F,0xB2,0x07,0xE2,0x15,0x44,0x1B,0x9B,0x19,0x6F,0xA1,0x57,0x59,0x46,0xC5}; +BMD_CONST REFIID IID_IDeckLinkAttributes = /* ABC11843-D966-44CB-96E2-A1CB5D3135C4 */ {0xAB,0xC1,0x18,0x43,0xD9,0x66,0x44,0xCB,0x96,0xE2,0xA1,0xCB,0x5D,0x31,0x35,0xC4}; +BMD_CONST REFIID IID_IDeckLinkKeyer = /* 89AFCAF5-65F8-421E-98F7-96FE5F5BFBA3 */ {0x89,0xAF,0xCA,0xF5,0x65,0xF8,0x42,0x1E,0x98,0xF7,0x96,0xFE,0x5F,0x5B,0xFB,0xA3}; +BMD_CONST REFIID IID_IDeckLinkVideoConversion = /* 3BBCB8A2-DA2C-42D9-B5D8-88083644E99A */ {0x3B,0xBC,0xB8,0xA2,0xDA,0x2C,0x42,0xD9,0xB5,0xD8,0x88,0x08,0x36,0x44,0xE9,0x9A}; +BMD_CONST REFIID IID_IDeckLinkDeviceNotificationCallback = /* 4997053B-0ADF-4CC8-AC70-7A50C4BE728F */ {0x49,0x97,0x05,0x3B,0x0A,0xDF,0x4C,0xC8,0xAC,0x70,0x7A,0x50,0xC4,0xBE,0x72,0x8F}; +BMD_CONST REFIID IID_IDeckLinkDiscovery = /* CDBF631C-BC76-45FA-B44D-C55059BC6101 */ {0xCD,0xBF,0x63,0x1C,0xBC,0x76,0x45,0xFA,0xB4,0x4D,0xC5,0x50,0x59,0xBC,0x61,0x01}; + +/* Enum BMDVideoOutputFlags - Flags to control the output of ancillary data along with video. */ + +typedef uint32_t BMDVideoOutputFlags; +enum _BMDVideoOutputFlags { + bmdVideoOutputFlagDefault = 0, + bmdVideoOutputVANC = 1 << 0, + bmdVideoOutputVITC = 1 << 1, + bmdVideoOutputRP188 = 1 << 2, + bmdVideoOutputDualStream3D = 1 << 4 +}; + +/* Enum BMDPacketType - Type of packet */ + +typedef uint32_t BMDPacketType; +enum _BMDPacketType { + bmdPacketTypeStreamInterruptedMarker = /* 'sint' */ 0x73696E74, // A packet of this type marks the time when a video stream was interrupted, for example by a disconnected cable + bmdPacketTypeStreamData = /* 'sdat' */ 0x73646174 // Regular stream data +}; + +/* Enum BMDFrameFlags - Frame flags */ + +typedef uint32_t BMDFrameFlags; +enum _BMDFrameFlags { + bmdFrameFlagDefault = 0, + bmdFrameFlagFlipVertical = 1 << 0, + + /* Flags that are applicable only to instances of IDeckLinkVideoInputFrame */ + + bmdFrameHasNoInputSource = 1 << 31 +}; + +/* Enum BMDVideoInputFlags - Flags applicable to video input */ + +typedef uint32_t BMDVideoInputFlags; +enum _BMDVideoInputFlags { + bmdVideoInputFlagDefault = 0, + bmdVideoInputEnableFormatDetection = 1 << 0, + bmdVideoInputDualStream3D = 1 << 1 +}; + +/* Enum BMDVideoInputFormatChangedEvents - Bitmask passed to the VideoInputFormatChanged notification to identify the properties of the input signal that have changed */ + +typedef uint32_t BMDVideoInputFormatChangedEvents; +enum _BMDVideoInputFormatChangedEvents { + bmdVideoInputDisplayModeChanged = 1 << 0, + bmdVideoInputFieldDominanceChanged = 1 << 1, + bmdVideoInputColorspaceChanged = 1 << 2 +}; + +/* Enum BMDDetectedVideoInputFormatFlags - Flags passed to the VideoInputFormatChanged notification to describe the detected video input signal */ + +typedef uint32_t BMDDetectedVideoInputFormatFlags; +enum _BMDDetectedVideoInputFormatFlags { + bmdDetectedVideoInputYCbCr422 = 1 << 0, + bmdDetectedVideoInputRGB444 = 1 << 1, + bmdDetectedVideoInputDualStream3D = 1 << 2 +}; + +/* Enum BMDDeckLinkCapturePassthroughMode - Enumerates whether the video output is electrically connected to the video input or if the clean switching mode is enabled */ + +typedef uint32_t BMDDeckLinkCapturePassthroughMode; +enum _BMDDeckLinkCapturePassthroughMode { + bmdDeckLinkCapturePassthroughModeDirect = /* 'pdir' */ 0x70646972, + bmdDeckLinkCapturePassthroughModeCleanSwitch = /* 'pcln' */ 0x70636C6E +}; + +/* Enum BMDOutputFrameCompletionResult - Frame Completion Callback */ + +typedef uint32_t BMDOutputFrameCompletionResult; +enum _BMDOutputFrameCompletionResult { + bmdOutputFrameCompleted, + bmdOutputFrameDisplayedLate, + bmdOutputFrameDropped, + bmdOutputFrameFlushed +}; + +/* Enum BMDReferenceStatus - GenLock input status */ + +typedef uint32_t BMDReferenceStatus; +enum _BMDReferenceStatus { + bmdReferenceNotSupportedByHardware = 1 << 0, + bmdReferenceLocked = 1 << 1 +}; + +/* Enum BMDAudioFormat - Audio Format */ + +typedef uint32_t BMDAudioFormat; +enum _BMDAudioFormat { + bmdAudioFormatPCM = /* 'lpcm' */ 0x6C70636D // Linear signed PCM samples +}; + +/* Enum BMDAudioSampleRate - Audio sample rates supported for output/input */ + +typedef uint32_t BMDAudioSampleRate; +enum _BMDAudioSampleRate { + bmdAudioSampleRate48kHz = 48000 +}; + +/* Enum BMDAudioSampleType - Audio sample sizes supported for output/input */ + +typedef uint32_t BMDAudioSampleType; +enum _BMDAudioSampleType { + bmdAudioSampleType16bitInteger = 16, + bmdAudioSampleType32bitInteger = 32 +}; + +/* Enum BMDAudioOutputStreamType - Audio output stream type */ + +typedef uint32_t BMDAudioOutputStreamType; +enum _BMDAudioOutputStreamType { + bmdAudioOutputStreamContinuous, + bmdAudioOutputStreamContinuousDontResample, + bmdAudioOutputStreamTimestamped +}; + +/* Enum BMDDisplayModeSupport - Output mode supported flags */ + +typedef uint32_t BMDDisplayModeSupport; +enum _BMDDisplayModeSupport { + bmdDisplayModeNotSupported = 0, + bmdDisplayModeSupported, + bmdDisplayModeSupportedWithConversion +}; + +/* Enum BMDTimecodeFormat - Timecode formats for frame metadata */ + +typedef uint32_t BMDTimecodeFormat; +enum _BMDTimecodeFormat { + bmdTimecodeRP188VITC1 = /* 'rpv1' */ 0x72707631, // RP188 timecode where DBB1 equals VITC1 (line 9) + bmdTimecodeRP188VITC2 = /* 'rp12' */ 0x72703132, // RP188 timecode where DBB1 equals VITC2 (line 9 for progressive or line 571 for interlaced/PsF) + bmdTimecodeRP188LTC = /* 'rplt' */ 0x72706C74, // RP188 timecode where DBB1 equals LTC (line 10) + bmdTimecodeRP188Any = /* 'rp18' */ 0x72703138, // For capture: return the first valid timecode in {VITC1, LTC ,VITC2} - For playback: set the timecode as VITC1 + bmdTimecodeVITC = /* 'vitc' */ 0x76697463, + bmdTimecodeVITCField2 = /* 'vit2' */ 0x76697432, + bmdTimecodeSerial = /* 'seri' */ 0x73657269 +}; + +/* Enum BMDAnalogVideoFlags - Analog video display flags */ + +typedef uint32_t BMDAnalogVideoFlags; +enum _BMDAnalogVideoFlags { + bmdAnalogVideoFlagCompositeSetup75 = 1 << 0, + bmdAnalogVideoFlagComponentBetacamLevels = 1 << 1 +}; + +/* Enum BMDAudioOutputAnalogAESSwitch - Audio output Analog/AESEBU switch */ + +typedef uint32_t BMDAudioOutputAnalogAESSwitch; +enum _BMDAudioOutputAnalogAESSwitch { + bmdAudioOutputSwitchAESEBU = /* 'aes ' */ 0x61657320, + bmdAudioOutputSwitchAnalog = /* 'anlg' */ 0x616E6C67 +}; + +/* Enum BMDVideoOutputConversionMode - Video/audio conversion mode */ + +typedef uint32_t BMDVideoOutputConversionMode; +enum _BMDVideoOutputConversionMode { + bmdNoVideoOutputConversion = /* 'none' */ 0x6E6F6E65, + bmdVideoOutputLetterboxDownconversion = /* 'ltbx' */ 0x6C746278, + bmdVideoOutputAnamorphicDownconversion = /* 'amph' */ 0x616D7068, + bmdVideoOutputHD720toHD1080Conversion = /* '720c' */ 0x37323063, + bmdVideoOutputHardwareLetterboxDownconversion = /* 'HWlb' */ 0x48576C62, + bmdVideoOutputHardwareAnamorphicDownconversion = /* 'HWam' */ 0x4857616D, + bmdVideoOutputHardwareCenterCutDownconversion = /* 'HWcc' */ 0x48576363, + bmdVideoOutputHardware720p1080pCrossconversion = /* 'xcap' */ 0x78636170, + bmdVideoOutputHardwareAnamorphic720pUpconversion = /* 'ua7p' */ 0x75613770, + bmdVideoOutputHardwareAnamorphic1080iUpconversion = /* 'ua1i' */ 0x75613169, + bmdVideoOutputHardwareAnamorphic149To720pUpconversion = /* 'u47p' */ 0x75343770, + bmdVideoOutputHardwareAnamorphic149To1080iUpconversion = /* 'u41i' */ 0x75343169, + bmdVideoOutputHardwarePillarbox720pUpconversion = /* 'up7p' */ 0x75703770, + bmdVideoOutputHardwarePillarbox1080iUpconversion = /* 'up1i' */ 0x75703169 +}; + +/* Enum BMDVideoInputConversionMode - Video input conversion mode */ + +typedef uint32_t BMDVideoInputConversionMode; +enum _BMDVideoInputConversionMode { + bmdNoVideoInputConversion = /* 'none' */ 0x6E6F6E65, + bmdVideoInputLetterboxDownconversionFromHD1080 = /* '10lb' */ 0x31306C62, + bmdVideoInputAnamorphicDownconversionFromHD1080 = /* '10am' */ 0x3130616D, + bmdVideoInputLetterboxDownconversionFromHD720 = /* '72lb' */ 0x37326C62, + bmdVideoInputAnamorphicDownconversionFromHD720 = /* '72am' */ 0x3732616D, + bmdVideoInputLetterboxUpconversion = /* 'lbup' */ 0x6C627570, + bmdVideoInputAnamorphicUpconversion = /* 'amup' */ 0x616D7570 +}; + +/* Enum BMDVideo3DPackingFormat - Video 3D packing format */ + +typedef uint32_t BMDVideo3DPackingFormat; +enum _BMDVideo3DPackingFormat { + bmdVideo3DPackingSidebySideHalf = /* 'sbsh' */ 0x73627368, + bmdVideo3DPackingLinebyLine = /* 'lbyl' */ 0x6C62796C, + bmdVideo3DPackingTopAndBottom = /* 'tabo' */ 0x7461626F, + bmdVideo3DPackingFramePacking = /* 'frpk' */ 0x6672706B, + bmdVideo3DPackingLeftOnly = /* 'left' */ 0x6C656674, + bmdVideo3DPackingRightOnly = /* 'righ' */ 0x72696768 +}; + +/* Enum BMDIdleVideoOutputOperation - Video output operation when not playing video */ + +typedef uint32_t BMDIdleVideoOutputOperation; +enum _BMDIdleVideoOutputOperation { + bmdIdleVideoOutputBlack = /* 'blac' */ 0x626C6163, + bmdIdleVideoOutputLastFrame = /* 'lafa' */ 0x6C616661, + bmdIdleVideoOutputDesktop = /* 'desk' */ 0x6465736B +}; + +/* Enum BMDVideoEncoderFrameCodingMode - Video frame coding mode */ + +typedef uint32_t BMDVideoEncoderFrameCodingMode; +enum _BMDVideoEncoderFrameCodingMode { + bmdVideoEncoderFrameCodingModeInter = /* 'inte' */ 0x696E7465, + bmdVideoEncoderFrameCodingModeIntra = /* 'intr' */ 0x696E7472 +}; + +/* Enum BMDDNxHRLevel - DNxHR Levels */ + +typedef uint32_t BMDDNxHRLevel; +enum _BMDDNxHRLevel { + bmdDNxHRLevelSQ = /* 'dnsq' */ 0x646E7371, + bmdDNxHRLevelLB = /* 'dnlb' */ 0x646E6C62, + bmdDNxHRLevelHQ = /* 'dnhq' */ 0x646E6871, + bmdDNxHRLevelHQX = /* 'dhqx' */ 0x64687178, + bmdDNxHRLevel444 = /* 'd444' */ 0x64343434 +}; + +/* Enum BMDLinkConfiguration - Video link configuration */ + +typedef uint32_t BMDLinkConfiguration; +enum _BMDLinkConfiguration { + bmdLinkConfigurationSingleLink = /* 'lcsl' */ 0x6C63736C, + bmdLinkConfigurationDualLink = /* 'lcdl' */ 0x6C63646C, + bmdLinkConfigurationQuadLink = /* 'lcql' */ 0x6C63716C +}; + +/* Enum BMDDeviceInterface - Device interface type */ + +typedef uint32_t BMDDeviceInterface; +enum _BMDDeviceInterface { + bmdDeviceInterfacePCI = /* 'pci ' */ 0x70636920, + bmdDeviceInterfaceUSB = /* 'usb ' */ 0x75736220, + bmdDeviceInterfaceThunderbolt = /* 'thun' */ 0x7468756E +}; + +/* Enum BMDDeckLinkAttributeID - DeckLink Attribute ID */ + +typedef uint32_t BMDDeckLinkAttributeID; +enum _BMDDeckLinkAttributeID { + + /* Flags */ + + BMDDeckLinkSupportsInternalKeying = /* 'keyi' */ 0x6B657969, + BMDDeckLinkSupportsExternalKeying = /* 'keye' */ 0x6B657965, + BMDDeckLinkSupportsHDKeying = /* 'keyh' */ 0x6B657968, + BMDDeckLinkSupportsInputFormatDetection = /* 'infd' */ 0x696E6664, + BMDDeckLinkHasReferenceInput = /* 'hrin' */ 0x6872696E, + BMDDeckLinkHasSerialPort = /* 'hspt' */ 0x68737074, + BMDDeckLinkHasAnalogVideoOutputGain = /* 'avog' */ 0x61766F67, + BMDDeckLinkCanOnlyAdjustOverallVideoOutputGain = /* 'ovog' */ 0x6F766F67, + BMDDeckLinkHasVideoInputAntiAliasingFilter = /* 'aafl' */ 0x6161666C, + BMDDeckLinkHasBypass = /* 'byps' */ 0x62797073, + BMDDeckLinkSupportsDesktopDisplay = /* 'extd' */ 0x65787464, + BMDDeckLinkSupportsClockTimingAdjustment = /* 'ctad' */ 0x63746164, + BMDDeckLinkSupportsFullDuplex = /* 'fdup' */ 0x66647570, + BMDDeckLinkSupportsFullFrameReferenceInputTimingOffset = /* 'frin' */ 0x6672696E, + BMDDeckLinkSupportsSMPTELevelAOutput = /* 'lvla' */ 0x6C766C61, + BMDDeckLinkSupportsDualLinkSDI = /* 'sdls' */ 0x73646C73, + BMDDeckLinkSupportsQuadLinkSDI = /* 'sqls' */ 0x73716C73, + BMDDeckLinkSupportsIdleOutput = /* 'idou' */ 0x69646F75, + BMDDeckLinkHasLTCTimecodeInput = /* 'hltc' */ 0x686C7463, + + /* Integers */ + + BMDDeckLinkMaximumAudioChannels = /* 'mach' */ 0x6D616368, + BMDDeckLinkMaximumAnalogAudioChannels = /* 'aach' */ 0x61616368, + BMDDeckLinkNumberOfSubDevices = /* 'nsbd' */ 0x6E736264, + BMDDeckLinkSubDeviceIndex = /* 'subi' */ 0x73756269, + BMDDeckLinkPersistentID = /* 'peid' */ 0x70656964, + BMDDeckLinkTopologicalID = /* 'toid' */ 0x746F6964, + BMDDeckLinkVideoOutputConnections = /* 'vocn' */ 0x766F636E, + BMDDeckLinkVideoInputConnections = /* 'vicn' */ 0x7669636E, + BMDDeckLinkAudioOutputConnections = /* 'aocn' */ 0x616F636E, + BMDDeckLinkAudioInputConnections = /* 'aicn' */ 0x6169636E, + BMDDeckLinkDeviceBusyState = /* 'dbst' */ 0x64627374, + BMDDeckLinkVideoIOSupport = /* 'vios' */ 0x76696F73, // Returns a BMDVideoIOSupport bit field + BMDDeckLinkDeckControlConnections = /* 'dccn' */ 0x6463636E, + BMDDeckLinkDeviceInterface = /* 'dbus' */ 0x64627573, // Returns a BMDDeviceInterface + BMDDeckLinkAudioInputRCAChannelCount = /* 'airc' */ 0x61697263, + BMDDeckLinkAudioInputXLRChannelCount = /* 'aixc' */ 0x61697863, + BMDDeckLinkAudioOutputRCAChannelCount = /* 'aorc' */ 0x616F7263, + BMDDeckLinkAudioOutputXLRChannelCount = /* 'aoxc' */ 0x616F7863, + + /* Floats */ + + BMDDeckLinkVideoInputGainMinimum = /* 'vigm' */ 0x7669676D, + BMDDeckLinkVideoInputGainMaximum = /* 'vigx' */ 0x76696778, + BMDDeckLinkVideoOutputGainMinimum = /* 'vogm' */ 0x766F676D, + BMDDeckLinkVideoOutputGainMaximum = /* 'vogx' */ 0x766F6778, + BMDDeckLinkMicrophoneInputGainMinimum = /* 'migm' */ 0x6D69676D, + BMDDeckLinkMicrophoneInputGainMaximum = /* 'migx' */ 0x6D696778, + + /* Strings */ + + BMDDeckLinkSerialPortDeviceName = /* 'slpn' */ 0x736C706E, + BMDDeckLinkVendorName = /* 'vndr' */ 0x766E6472, + BMDDeckLinkDisplayName = /* 'dspn' */ 0x6473706E, + BMDDeckLinkModelName = /* 'mdln' */ 0x6D646C6E +}; + +/* Enum BMDDeckLinkAPIInformationID - DeckLinkAPI information ID */ + +typedef uint32_t BMDDeckLinkAPIInformationID; +enum _BMDDeckLinkAPIInformationID { + BMDDeckLinkAPIVersion = /* 'vers' */ 0x76657273 +}; + +/* Enum BMDDeviceBusyState - Current device busy state */ + +typedef uint32_t BMDDeviceBusyState; +enum _BMDDeviceBusyState { + bmdDeviceCaptureBusy = 1 << 0, + bmdDevicePlaybackBusy = 1 << 1, + bmdDeviceSerialPortBusy = 1 << 2 +}; + +/* Enum BMDVideoIOSupport - Device video input/output support */ + +typedef uint32_t BMDVideoIOSupport; +enum _BMDVideoIOSupport { + bmdDeviceSupportsCapture = 1 << 0, + bmdDeviceSupportsPlayback = 1 << 1 +}; + +/* Enum BMD3DPreviewFormat - Linked Frame preview format */ + +typedef uint32_t BMD3DPreviewFormat; +enum _BMD3DPreviewFormat { + bmd3DPreviewFormatDefault = /* 'defa' */ 0x64656661, + bmd3DPreviewFormatLeftOnly = /* 'left' */ 0x6C656674, + bmd3DPreviewFormatRightOnly = /* 'righ' */ 0x72696768, + bmd3DPreviewFormatSideBySide = /* 'side' */ 0x73696465, + bmd3DPreviewFormatTopBottom = /* 'topb' */ 0x746F7062 +}; + +/* Enum BMDNotifications - Events that can be subscribed through IDeckLinkNotification */ + +typedef uint32_t BMDNotifications; +enum _BMDNotifications { + bmdPreferencesChanged = /* 'pref' */ 0x70726566 +}; + +#if defined(__cplusplus) + +// Forward Declarations + +class IDeckLinkVideoOutputCallback; +class IDeckLinkInputCallback; +class IDeckLinkEncoderInputCallback; +class IDeckLinkMemoryAllocator; +class IDeckLinkAudioOutputCallback; +class IDeckLinkIterator; +class IDeckLinkAPIInformation; +class IDeckLinkOutput; +class IDeckLinkInput; +class IDeckLinkEncoderInput; +class IDeckLinkVideoFrame; +class IDeckLinkMutableVideoFrame; +class IDeckLinkVideoFrame3DExtensions; +class IDeckLinkVideoInputFrame; +class IDeckLinkVideoFrameAncillary; +class IDeckLinkEncoderPacket; +class IDeckLinkEncoderVideoPacket; +class IDeckLinkEncoderAudioPacket; +class IDeckLinkH265NALPacket; +class IDeckLinkAudioInputPacket; +class IDeckLinkScreenPreviewCallback; +class IDeckLinkGLScreenPreviewHelper; +class IDeckLinkNotificationCallback; +class IDeckLinkNotification; +class IDeckLinkAttributes; +class IDeckLinkKeyer; +class IDeckLinkVideoConversion; +class IDeckLinkDeviceNotificationCallback; +class IDeckLinkDiscovery; + +/* Interface IDeckLinkVideoOutputCallback - Frame completion callback. */ + +class IDeckLinkVideoOutputCallback : public IUnknown +{ +public: + virtual HRESULT ScheduledFrameCompleted (/* in */ IDeckLinkVideoFrame *completedFrame, /* in */ BMDOutputFrameCompletionResult result) = 0; + virtual HRESULT ScheduledPlaybackHasStopped (void) = 0; + +protected: + virtual ~IDeckLinkVideoOutputCallback () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkInputCallback - Frame arrival callback. */ + +class IDeckLinkInputCallback : public IUnknown +{ +public: + virtual HRESULT VideoInputFormatChanged (/* in */ BMDVideoInputFormatChangedEvents notificationEvents, /* in */ IDeckLinkDisplayMode *newDisplayMode, /* in */ BMDDetectedVideoInputFormatFlags detectedSignalFlags) = 0; + virtual HRESULT VideoInputFrameArrived (/* in */ IDeckLinkVideoInputFrame* videoFrame, /* in */ IDeckLinkAudioInputPacket* audioPacket) = 0; + +protected: + virtual ~IDeckLinkInputCallback () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkEncoderInputCallback - Frame arrival callback. */ + +class IDeckLinkEncoderInputCallback : public IUnknown +{ +public: + virtual HRESULT VideoInputSignalChanged (/* in */ BMDVideoInputFormatChangedEvents notificationEvents, /* in */ IDeckLinkDisplayMode *newDisplayMode, /* in */ BMDDetectedVideoInputFormatFlags detectedSignalFlags) = 0; + virtual HRESULT VideoPacketArrived (/* in */ IDeckLinkEncoderVideoPacket* videoPacket) = 0; + virtual HRESULT AudioPacketArrived (/* in */ IDeckLinkEncoderAudioPacket* audioPacket) = 0; + +protected: + virtual ~IDeckLinkEncoderInputCallback () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkMemoryAllocator - Memory allocator for video frames. */ + +class IDeckLinkMemoryAllocator : public IUnknown +{ +public: + virtual HRESULT AllocateBuffer (/* in */ uint32_t bufferSize, /* out */ void **allocatedBuffer) = 0; + virtual HRESULT ReleaseBuffer (/* in */ void *buffer) = 0; + + virtual HRESULT Commit (void) = 0; + virtual HRESULT Decommit (void) = 0; +}; + +/* Interface IDeckLinkAudioOutputCallback - Optional callback to allow audio samples to be pulled as required. */ + +class IDeckLinkAudioOutputCallback : public IUnknown +{ +public: + virtual HRESULT RenderAudioSamples (/* in */ bool preroll) = 0; +}; + +/* Interface IDeckLinkIterator - enumerates installed DeckLink hardware */ + +class IDeckLinkIterator : public IUnknown +{ +public: + virtual HRESULT Next (/* out */ IDeckLink **deckLinkInstance) = 0; +}; + +/* Interface IDeckLinkAPIInformation - DeckLinkAPI attribute interface */ + +class IDeckLinkAPIInformation : public IUnknown +{ +public: + virtual HRESULT GetFlag (/* in */ BMDDeckLinkAPIInformationID cfgID, /* out */ bool *value) = 0; + virtual HRESULT GetInt (/* in */ BMDDeckLinkAPIInformationID cfgID, /* out */ int64_t *value) = 0; + virtual HRESULT GetFloat (/* in */ BMDDeckLinkAPIInformationID cfgID, /* out */ double *value) = 0; + virtual HRESULT GetString (/* in */ BMDDeckLinkAPIInformationID cfgID, /* out */ const char **value) = 0; + +protected: + virtual ~IDeckLinkAPIInformation () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkOutput - Created by QueryInterface from IDeckLink. */ + +class IDeckLinkOutput : public IUnknown +{ +public: + virtual HRESULT DoesSupportVideoMode (/* in */ BMDDisplayMode displayMode, /* in */ BMDPixelFormat pixelFormat, /* in */ BMDVideoOutputFlags flags, /* out */ BMDDisplayModeSupport *result, /* out */ IDeckLinkDisplayMode **resultDisplayMode) = 0; + virtual HRESULT GetDisplayModeIterator (/* out */ IDeckLinkDisplayModeIterator **iterator) = 0; + + virtual HRESULT SetScreenPreviewCallback (/* in */ IDeckLinkScreenPreviewCallback *previewCallback) = 0; + + /* Video Output */ + + virtual HRESULT EnableVideoOutput (/* in */ BMDDisplayMode displayMode, /* in */ BMDVideoOutputFlags flags) = 0; + virtual HRESULT DisableVideoOutput (void) = 0; + + virtual HRESULT SetVideoOutputFrameMemoryAllocator (/* in */ IDeckLinkMemoryAllocator *theAllocator) = 0; + virtual HRESULT CreateVideoFrame (/* in */ int32_t width, /* in */ int32_t height, /* in */ int32_t rowBytes, /* in */ BMDPixelFormat pixelFormat, /* in */ BMDFrameFlags flags, /* out */ IDeckLinkMutableVideoFrame **outFrame) = 0; + virtual HRESULT CreateAncillaryData (/* in */ BMDPixelFormat pixelFormat, /* out */ IDeckLinkVideoFrameAncillary **outBuffer) = 0; + + virtual HRESULT DisplayVideoFrameSync (/* in */ IDeckLinkVideoFrame *theFrame) = 0; + virtual HRESULT ScheduleVideoFrame (/* in */ IDeckLinkVideoFrame *theFrame, /* in */ BMDTimeValue displayTime, /* in */ BMDTimeValue displayDuration, /* in */ BMDTimeScale timeScale) = 0; + virtual HRESULT SetScheduledFrameCompletionCallback (/* in */ IDeckLinkVideoOutputCallback *theCallback) = 0; + virtual HRESULT GetBufferedVideoFrameCount (/* out */ uint32_t *bufferedFrameCount) = 0; + + /* Audio Output */ + + virtual HRESULT EnableAudioOutput (/* in */ BMDAudioSampleRate sampleRate, /* in */ BMDAudioSampleType sampleType, /* in */ uint32_t channelCount, /* in */ BMDAudioOutputStreamType streamType) = 0; + virtual HRESULT DisableAudioOutput (void) = 0; + + virtual HRESULT WriteAudioSamplesSync (/* in */ void *buffer, /* in */ uint32_t sampleFrameCount, /* out */ uint32_t *sampleFramesWritten) = 0; + + virtual HRESULT BeginAudioPreroll (void) = 0; + virtual HRESULT EndAudioPreroll (void) = 0; + virtual HRESULT ScheduleAudioSamples (/* in */ void *buffer, /* in */ uint32_t sampleFrameCount, /* in */ BMDTimeValue streamTime, /* in */ BMDTimeScale timeScale, /* out */ uint32_t *sampleFramesWritten) = 0; + + virtual HRESULT GetBufferedAudioSampleFrameCount (/* out */ uint32_t *bufferedSampleFrameCount) = 0; + virtual HRESULT FlushBufferedAudioSamples (void) = 0; + + virtual HRESULT SetAudioCallback (/* in */ IDeckLinkAudioOutputCallback *theCallback) = 0; + + /* Output Control */ + + virtual HRESULT StartScheduledPlayback (/* in */ BMDTimeValue playbackStartTime, /* in */ BMDTimeScale timeScale, /* in */ double playbackSpeed) = 0; + virtual HRESULT StopScheduledPlayback (/* in */ BMDTimeValue stopPlaybackAtTime, /* out */ BMDTimeValue *actualStopTime, /* in */ BMDTimeScale timeScale) = 0; + virtual HRESULT IsScheduledPlaybackRunning (/* out */ bool *active) = 0; + virtual HRESULT GetScheduledStreamTime (/* in */ BMDTimeScale desiredTimeScale, /* out */ BMDTimeValue *streamTime, /* out */ double *playbackSpeed) = 0; + virtual HRESULT GetReferenceStatus (/* out */ BMDReferenceStatus *referenceStatus) = 0; + + /* Hardware Timing */ + + virtual HRESULT GetHardwareReferenceClock (/* in */ BMDTimeScale desiredTimeScale, /* out */ BMDTimeValue *hardwareTime, /* out */ BMDTimeValue *timeInFrame, /* out */ BMDTimeValue *ticksPerFrame) = 0; + virtual HRESULT GetFrameCompletionReferenceTimestamp (/* in */ IDeckLinkVideoFrame *theFrame, /* in */ BMDTimeScale desiredTimeScale, /* out */ BMDTimeValue *frameCompletionTimestamp) = 0; + +protected: + virtual ~IDeckLinkOutput () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkInput - Created by QueryInterface from IDeckLink. */ + +class IDeckLinkInput : public IUnknown +{ +public: + virtual HRESULT DoesSupportVideoMode (/* in */ BMDDisplayMode displayMode, /* in */ BMDPixelFormat pixelFormat, /* in */ BMDVideoInputFlags flags, /* out */ BMDDisplayModeSupport *result, /* out */ IDeckLinkDisplayMode **resultDisplayMode) = 0; + virtual HRESULT GetDisplayModeIterator (/* out */ IDeckLinkDisplayModeIterator **iterator) = 0; + + virtual HRESULT SetScreenPreviewCallback (/* in */ IDeckLinkScreenPreviewCallback *previewCallback) = 0; + + /* Video Input */ + + virtual HRESULT EnableVideoInput (/* in */ BMDDisplayMode displayMode, /* in */ BMDPixelFormat pixelFormat, /* in */ BMDVideoInputFlags flags) = 0; + virtual HRESULT DisableVideoInput (void) = 0; + virtual HRESULT GetAvailableVideoFrameCount (/* out */ uint32_t *availableFrameCount) = 0; + virtual HRESULT SetVideoInputFrameMemoryAllocator (/* in */ IDeckLinkMemoryAllocator *theAllocator) = 0; + + /* Audio Input */ + + virtual HRESULT EnableAudioInput (/* in */ BMDAudioSampleRate sampleRate, /* in */ BMDAudioSampleType sampleType, /* in */ uint32_t channelCount) = 0; + virtual HRESULT DisableAudioInput (void) = 0; + virtual HRESULT GetAvailableAudioSampleFrameCount (/* out */ uint32_t *availableSampleFrameCount) = 0; + + /* Input Control */ + + virtual HRESULT StartStreams (void) = 0; + virtual HRESULT StopStreams (void) = 0; + virtual HRESULT PauseStreams (void) = 0; + virtual HRESULT FlushStreams (void) = 0; + virtual HRESULT SetCallback (/* in */ IDeckLinkInputCallback *theCallback) = 0; + + /* Hardware Timing */ + + virtual HRESULT GetHardwareReferenceClock (/* in */ BMDTimeScale desiredTimeScale, /* out */ BMDTimeValue *hardwareTime, /* out */ BMDTimeValue *timeInFrame, /* out */ BMDTimeValue *ticksPerFrame) = 0; + +protected: + virtual ~IDeckLinkInput () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkEncoderInput - Created by QueryInterface from IDeckLink. */ + +class IDeckLinkEncoderInput : public IUnknown +{ +public: + virtual HRESULT DoesSupportVideoMode (/* in */ BMDDisplayMode displayMode, /* in */ BMDPixelFormat pixelFormat, /* in */ BMDVideoInputFlags flags, /* out */ BMDDisplayModeSupport *result, /* out */ IDeckLinkDisplayMode **resultDisplayMode) = 0; + virtual HRESULT GetDisplayModeIterator (/* out */ IDeckLinkDisplayModeIterator **iterator) = 0; + + /* Video Input */ + + virtual HRESULT EnableVideoInput (/* in */ BMDDisplayMode displayMode, /* in */ BMDPixelFormat pixelFormat, /* in */ BMDVideoInputFlags flags) = 0; + virtual HRESULT DisableVideoInput (void) = 0; + virtual HRESULT GetAvailablePacketsCount (/* out */ uint32_t *availablePacketsCount) = 0; + virtual HRESULT SetMemoryAllocator (/* in */ IDeckLinkMemoryAllocator *theAllocator) = 0; + + /* Audio Input */ + + virtual HRESULT EnableAudioInput (/* in */ BMDAudioFormat audioFormat, /* in */ BMDAudioSampleRate sampleRate, /* in */ BMDAudioSampleType sampleType, /* in */ uint32_t channelCount) = 0; + virtual HRESULT DisableAudioInput (void) = 0; + virtual HRESULT GetAvailableAudioSampleFrameCount (/* out */ uint32_t *availableSampleFrameCount) = 0; + + /* Input Control */ + + virtual HRESULT StartStreams (void) = 0; + virtual HRESULT StopStreams (void) = 0; + virtual HRESULT PauseStreams (void) = 0; + virtual HRESULT FlushStreams (void) = 0; + virtual HRESULT SetCallback (/* in */ IDeckLinkEncoderInputCallback *theCallback) = 0; + + /* Hardware Timing */ + + virtual HRESULT GetHardwareReferenceClock (/* in */ BMDTimeScale desiredTimeScale, /* out */ BMDTimeValue *hardwareTime, /* out */ BMDTimeValue *timeInFrame, /* out */ BMDTimeValue *ticksPerFrame) = 0; + +protected: + virtual ~IDeckLinkEncoderInput () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkVideoFrame - Interface to encapsulate a video frame; can be caller-implemented. */ + +class IDeckLinkVideoFrame : public IUnknown +{ +public: + virtual long GetWidth (void) = 0; + virtual long GetHeight (void) = 0; + virtual long GetRowBytes (void) = 0; + virtual BMDPixelFormat GetPixelFormat (void) = 0; + virtual BMDFrameFlags GetFlags (void) = 0; + virtual HRESULT GetBytes (/* out */ void **buffer) = 0; + + virtual HRESULT GetTimecode (/* in */ BMDTimecodeFormat format, /* out */ IDeckLinkTimecode **timecode) = 0; + virtual HRESULT GetAncillaryData (/* out */ IDeckLinkVideoFrameAncillary **ancillary) = 0; + +protected: + virtual ~IDeckLinkVideoFrame () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkMutableVideoFrame - Created by IDeckLinkOutput::CreateVideoFrame. */ + +class IDeckLinkMutableVideoFrame : public IDeckLinkVideoFrame +{ +public: + virtual HRESULT SetFlags (/* in */ BMDFrameFlags newFlags) = 0; + + virtual HRESULT SetTimecode (/* in */ BMDTimecodeFormat format, /* in */ IDeckLinkTimecode *timecode) = 0; + virtual HRESULT SetTimecodeFromComponents (/* in */ BMDTimecodeFormat format, /* in */ uint8_t hours, /* in */ uint8_t minutes, /* in */ uint8_t seconds, /* in */ uint8_t frames, /* in */ BMDTimecodeFlags flags) = 0; + virtual HRESULT SetAncillaryData (/* in */ IDeckLinkVideoFrameAncillary *ancillary) = 0; + virtual HRESULT SetTimecodeUserBits (/* in */ BMDTimecodeFormat format, /* in */ BMDTimecodeUserBits userBits) = 0; + +protected: + virtual ~IDeckLinkMutableVideoFrame () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkVideoFrame3DExtensions - Optional interface implemented on IDeckLinkVideoFrame to support 3D frames */ + +class IDeckLinkVideoFrame3DExtensions : public IUnknown +{ +public: + virtual BMDVideo3DPackingFormat Get3DPackingFormat (void) = 0; + virtual HRESULT GetFrameForRightEye (/* out */ IDeckLinkVideoFrame* *rightEyeFrame) = 0; + +protected: + virtual ~IDeckLinkVideoFrame3DExtensions () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkVideoInputFrame - Provided by the IDeckLinkVideoInput frame arrival callback. */ + +class IDeckLinkVideoInputFrame : public IDeckLinkVideoFrame +{ +public: + virtual HRESULT GetStreamTime (/* out */ BMDTimeValue *frameTime, /* out */ BMDTimeValue *frameDuration, /* in */ BMDTimeScale timeScale) = 0; + virtual HRESULT GetHardwareReferenceTimestamp (/* in */ BMDTimeScale timeScale, /* out */ BMDTimeValue *frameTime, /* out */ BMDTimeValue *frameDuration) = 0; + +protected: + virtual ~IDeckLinkVideoInputFrame () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkVideoFrameAncillary - Obtained through QueryInterface() on an IDeckLinkVideoFrame object. */ + +class IDeckLinkVideoFrameAncillary : public IUnknown +{ +public: + + virtual HRESULT GetBufferForVerticalBlankingLine (/* in */ uint32_t lineNumber, /* out */ void **buffer) = 0; + virtual BMDPixelFormat GetPixelFormat (void) = 0; + virtual BMDDisplayMode GetDisplayMode (void) = 0; + +protected: + virtual ~IDeckLinkVideoFrameAncillary () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkEncoderPacket - Interface to encapsulate an encoded packet. */ + +class IDeckLinkEncoderPacket : public IUnknown +{ +public: + virtual HRESULT GetBytes (/* out */ void **buffer) = 0; + virtual long GetSize (void) = 0; + virtual HRESULT GetStreamTime (/* out */ BMDTimeValue *frameTime, /* in */ BMDTimeScale timeScale) = 0; + virtual BMDPacketType GetPacketType (void) = 0; + +protected: + virtual ~IDeckLinkEncoderPacket () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkEncoderVideoPacket - Provided by the IDeckLinkEncoderInput video packet arrival callback. */ + +class IDeckLinkEncoderVideoPacket : public IDeckLinkEncoderPacket +{ +public: + virtual BMDPixelFormat GetPixelFormat (void) = 0; + virtual HRESULT GetHardwareReferenceTimestamp (/* in */ BMDTimeScale timeScale, /* out */ BMDTimeValue *frameTime, /* out */ BMDTimeValue *frameDuration) = 0; + + virtual HRESULT GetTimecode (/* in */ BMDTimecodeFormat format, /* out */ IDeckLinkTimecode **timecode) = 0; + +protected: + virtual ~IDeckLinkEncoderVideoPacket () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkEncoderAudioPacket - Provided by the IDeckLinkEncoderInput audio packet arrival callback. */ + +class IDeckLinkEncoderAudioPacket : public IDeckLinkEncoderPacket +{ +public: + virtual BMDAudioFormat GetAudioFormat (void) = 0; + +protected: + virtual ~IDeckLinkEncoderAudioPacket () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkH265NALPacket - Obtained through QueryInterface() on an IDeckLinkEncoderVideoPacket object */ + +class IDeckLinkH265NALPacket : public IDeckLinkEncoderVideoPacket +{ +public: + virtual HRESULT GetUnitType (/* out */ uint8_t *unitType) = 0; + virtual HRESULT GetBytesNoPrefix (/* out */ void **buffer) = 0; + virtual long GetSizeNoPrefix (void) = 0; + +protected: + virtual ~IDeckLinkH265NALPacket () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkAudioInputPacket - Provided by the IDeckLinkInput callback. */ + +class IDeckLinkAudioInputPacket : public IUnknown +{ +public: + virtual long GetSampleFrameCount (void) = 0; + virtual HRESULT GetBytes (/* out */ void **buffer) = 0; + virtual HRESULT GetPacketTime (/* out */ BMDTimeValue *packetTime, /* in */ BMDTimeScale timeScale) = 0; + +protected: + virtual ~IDeckLinkAudioInputPacket () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkScreenPreviewCallback - Screen preview callback */ + +class IDeckLinkScreenPreviewCallback : public IUnknown +{ +public: + virtual HRESULT DrawFrame (/* in */ IDeckLinkVideoFrame *theFrame) = 0; + +protected: + virtual ~IDeckLinkScreenPreviewCallback () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkGLScreenPreviewHelper - Created with CoCreateInstance(). */ + +class IDeckLinkGLScreenPreviewHelper : public IUnknown +{ +public: + + /* Methods must be called with OpenGL context set */ + + virtual HRESULT InitializeGL (void) = 0; + virtual HRESULT PaintGL (void) = 0; + virtual HRESULT SetFrame (/* in */ IDeckLinkVideoFrame *theFrame) = 0; + virtual HRESULT Set3DPreviewFormat (/* in */ BMD3DPreviewFormat previewFormat) = 0; + +protected: + virtual ~IDeckLinkGLScreenPreviewHelper () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkNotificationCallback - DeckLink Notification Callback Interface */ + +class IDeckLinkNotificationCallback : public IUnknown +{ +public: + virtual HRESULT Notify (/* in */ BMDNotifications topic, /* in */ uint64_t param1, /* in */ uint64_t param2) = 0; +}; + +/* Interface IDeckLinkNotification - DeckLink Notification interface */ + +class IDeckLinkNotification : public IUnknown +{ +public: + virtual HRESULT Subscribe (/* in */ BMDNotifications topic, /* in */ IDeckLinkNotificationCallback *theCallback) = 0; + virtual HRESULT Unsubscribe (/* in */ BMDNotifications topic, /* in */ IDeckLinkNotificationCallback *theCallback) = 0; +}; + +/* Interface IDeckLinkAttributes - DeckLink Attribute interface */ + +class IDeckLinkAttributes : public IUnknown +{ +public: + virtual HRESULT GetFlag (/* in */ BMDDeckLinkAttributeID cfgID, /* out */ bool *value) = 0; + virtual HRESULT GetInt (/* in */ BMDDeckLinkAttributeID cfgID, /* out */ int64_t *value) = 0; + virtual HRESULT GetFloat (/* in */ BMDDeckLinkAttributeID cfgID, /* out */ double *value) = 0; + virtual HRESULT GetString (/* in */ BMDDeckLinkAttributeID cfgID, /* out */ const char **value) = 0; + +protected: + virtual ~IDeckLinkAttributes () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkKeyer - DeckLink Keyer interface */ + +class IDeckLinkKeyer : public IUnknown +{ +public: + virtual HRESULT Enable (/* in */ bool isExternal) = 0; + virtual HRESULT SetLevel (/* in */ uint8_t level) = 0; + virtual HRESULT RampUp (/* in */ uint32_t numberOfFrames) = 0; + virtual HRESULT RampDown (/* in */ uint32_t numberOfFrames) = 0; + virtual HRESULT Disable (void) = 0; + +protected: + virtual ~IDeckLinkKeyer () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkVideoConversion - Created with CoCreateInstance(). */ + +class IDeckLinkVideoConversion : public IUnknown +{ +public: + virtual HRESULT ConvertFrame (/* in */ IDeckLinkVideoFrame* srcFrame, /* in */ IDeckLinkVideoFrame* dstFrame) = 0; + +protected: + virtual ~IDeckLinkVideoConversion () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkDeviceNotificationCallback - DeckLink device arrival/removal notification callbacks */ + +class IDeckLinkDeviceNotificationCallback : public IUnknown +{ +public: + virtual HRESULT DeckLinkDeviceArrived (/* in */ IDeckLink* deckLinkDevice) = 0; + virtual HRESULT DeckLinkDeviceRemoved (/* in */ IDeckLink* deckLinkDevice) = 0; + +protected: + virtual ~IDeckLinkDeviceNotificationCallback () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkDiscovery - DeckLink device discovery */ + +class IDeckLinkDiscovery : public IUnknown +{ +public: + virtual HRESULT InstallDeviceNotifications (/* in */ IDeckLinkDeviceNotificationCallback* deviceNotificationCallback) = 0; + virtual HRESULT UninstallDeviceNotifications (void) = 0; + +protected: + virtual ~IDeckLinkDiscovery () {} // call Release method to drop reference count +}; + +/* Functions */ + +extern "C" { + + IDeckLinkIterator* CreateDeckLinkIteratorInstance (void); + IDeckLinkDiscovery* CreateDeckLinkDiscoveryInstance (void); + IDeckLinkAPIInformation* CreateDeckLinkAPIInformationInstance (void); + IDeckLinkGLScreenPreviewHelper* CreateOpenGLScreenPreviewHelper (void); + IDeckLinkVideoConversion* CreateVideoConversionInstance (void); + +} + + +#endif // defined(__cplusplus) +#endif /* defined(BMD_DECKLINKAPI_H) */ diff --git a/nageru/decklink/DeckLinkAPIConfiguration.h b/nageru/decklink/DeckLinkAPIConfiguration.h new file mode 100644 index 0000000..cac0e2f --- /dev/null +++ b/nageru/decklink/DeckLinkAPIConfiguration.h @@ -0,0 +1,252 @@ +/* -LICENSE-START- +** Copyright (c) 2015 Blackmagic Design +** +** Permission is hereby granted, free of charge, to any person or organization +** obtaining a copy of the software and accompanying documentation covered by +** this license (the "Software") to use, reproduce, display, distribute, +** execute, and transmit the Software, and to prepare derivative works of the +** Software, and to permit third-parties to whom the Software is furnished to +** do so, all subject to the following: +** +** The copyright notices in the Software and this entire statement, including +** the above license grant, this restriction and the following disclaimer, +** must be included in all copies of the Software, in whole or in part, and +** all derivative works of the Software, unless such copies or derivative +** works are solely in the form of machine-executable object code generated by +** a source language processor. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +** -LICENSE-END- +*/ + +#ifndef BMD_DECKLINKAPICONFIGURATION_H +#define BMD_DECKLINKAPICONFIGURATION_H + + +#ifndef BMD_CONST + #if defined(_MSC_VER) + #define BMD_CONST __declspec(selectany) static const + #else + #define BMD_CONST static const + #endif +#endif + +// Type Declarations + + +// Interface ID Declarations + +BMD_CONST REFIID IID_IDeckLinkConfiguration = /* CB71734A-FE37-4E8D-8E13-802133A1C3F2 */ {0xCB,0x71,0x73,0x4A,0xFE,0x37,0x4E,0x8D,0x8E,0x13,0x80,0x21,0x33,0xA1,0xC3,0xF2}; +BMD_CONST REFIID IID_IDeckLinkEncoderConfiguration = /* 138050E5-C60A-4552-BF3F-0F358049327E */ {0x13,0x80,0x50,0xE5,0xC6,0x0A,0x45,0x52,0xBF,0x3F,0x0F,0x35,0x80,0x49,0x32,0x7E}; + +/* Enum BMDDeckLinkConfigurationID - DeckLink Configuration ID */ + +typedef uint32_t BMDDeckLinkConfigurationID; +enum _BMDDeckLinkConfigurationID { + + /* Serial port Flags */ + + bmdDeckLinkConfigSwapSerialRxTx = /* 'ssrt' */ 0x73737274, + + /* Video Input/Output Flags */ + + bmdDeckLinkConfigUse1080pNotPsF = /* 'fpro' */ 0x6670726F, + + /* Video Input/Output Integers */ + + bmdDeckLinkConfigHDMI3DPackingFormat = /* '3dpf' */ 0x33647066, + bmdDeckLinkConfigBypass = /* 'byps' */ 0x62797073, + bmdDeckLinkConfigClockTimingAdjustment = /* 'ctad' */ 0x63746164, + + /* Audio Input/Output Flags */ + + bmdDeckLinkConfigAnalogAudioConsumerLevels = /* 'aacl' */ 0x6161636C, + + /* Video output flags */ + + bmdDeckLinkConfigFieldFlickerRemoval = /* 'fdfr' */ 0x66646672, + bmdDeckLinkConfigHD1080p24ToHD1080i5994Conversion = /* 'to59' */ 0x746F3539, + bmdDeckLinkConfig444SDIVideoOutput = /* '444o' */ 0x3434346F, + bmdDeckLinkConfigBlackVideoOutputDuringCapture = /* 'bvoc' */ 0x62766F63, + bmdDeckLinkConfigLowLatencyVideoOutput = /* 'llvo' */ 0x6C6C766F, + bmdDeckLinkConfigDownConversionOnAllAnalogOutput = /* 'caao' */ 0x6361616F, + bmdDeckLinkConfigSMPTELevelAOutput = /* 'smta' */ 0x736D7461, + + /* Video Output Integers */ + + bmdDeckLinkConfigVideoOutputConnection = /* 'vocn' */ 0x766F636E, + bmdDeckLinkConfigVideoOutputConversionMode = /* 'vocm' */ 0x766F636D, + bmdDeckLinkConfigAnalogVideoOutputFlags = /* 'avof' */ 0x61766F66, + bmdDeckLinkConfigReferenceInputTimingOffset = /* 'glot' */ 0x676C6F74, + bmdDeckLinkConfigVideoOutputIdleOperation = /* 'voio' */ 0x766F696F, + bmdDeckLinkConfigDefaultVideoOutputMode = /* 'dvom' */ 0x64766F6D, + bmdDeckLinkConfigDefaultVideoOutputModeFlags = /* 'dvof' */ 0x64766F66, + bmdDeckLinkConfigSDIOutputLinkConfiguration = /* 'solc' */ 0x736F6C63, + + /* Video Output Floats */ + + bmdDeckLinkConfigVideoOutputComponentLumaGain = /* 'oclg' */ 0x6F636C67, + bmdDeckLinkConfigVideoOutputComponentChromaBlueGain = /* 'occb' */ 0x6F636362, + bmdDeckLinkConfigVideoOutputComponentChromaRedGain = /* 'occr' */ 0x6F636372, + bmdDeckLinkConfigVideoOutputCompositeLumaGain = /* 'oilg' */ 0x6F696C67, + bmdDeckLinkConfigVideoOutputCompositeChromaGain = /* 'oicg' */ 0x6F696367, + bmdDeckLinkConfigVideoOutputSVideoLumaGain = /* 'oslg' */ 0x6F736C67, + bmdDeckLinkConfigVideoOutputSVideoChromaGain = /* 'oscg' */ 0x6F736367, + + /* Video Input Flags */ + + bmdDeckLinkConfigVideoInputScanning = /* 'visc' */ 0x76697363, // Applicable to H264 Pro Recorder only + bmdDeckLinkConfigUseDedicatedLTCInput = /* 'dltc' */ 0x646C7463, // Use timecode from LTC input instead of SDI stream + bmdDeckLinkConfigSDIInput3DPayloadOverride = /* '3dds' */ 0x33646473, + + /* Video Input Integers */ + + bmdDeckLinkConfigVideoInputConnection = /* 'vicn' */ 0x7669636E, + bmdDeckLinkConfigAnalogVideoInputFlags = /* 'avif' */ 0x61766966, + bmdDeckLinkConfigVideoInputConversionMode = /* 'vicm' */ 0x7669636D, + bmdDeckLinkConfig32PulldownSequenceInitialTimecodeFrame = /* 'pdif' */ 0x70646966, + bmdDeckLinkConfigVANCSourceLine1Mapping = /* 'vsl1' */ 0x76736C31, + bmdDeckLinkConfigVANCSourceLine2Mapping = /* 'vsl2' */ 0x76736C32, + bmdDeckLinkConfigVANCSourceLine3Mapping = /* 'vsl3' */ 0x76736C33, + bmdDeckLinkConfigCapturePassThroughMode = /* 'cptm' */ 0x6370746D, + + /* Video Input Floats */ + + bmdDeckLinkConfigVideoInputComponentLumaGain = /* 'iclg' */ 0x69636C67, + bmdDeckLinkConfigVideoInputComponentChromaBlueGain = /* 'iccb' */ 0x69636362, + bmdDeckLinkConfigVideoInputComponentChromaRedGain = /* 'iccr' */ 0x69636372, + bmdDeckLinkConfigVideoInputCompositeLumaGain = /* 'iilg' */ 0x69696C67, + bmdDeckLinkConfigVideoInputCompositeChromaGain = /* 'iicg' */ 0x69696367, + bmdDeckLinkConfigVideoInputSVideoLumaGain = /* 'islg' */ 0x69736C67, + bmdDeckLinkConfigVideoInputSVideoChromaGain = /* 'iscg' */ 0x69736367, + + /* Audio Input Flags */ + + bmdDeckLinkConfigMicrophonePhantomPower = /* 'mphp' */ 0x6D706870, + + /* Audio Input Integers */ + + bmdDeckLinkConfigAudioInputConnection = /* 'aicn' */ 0x6169636E, + + /* Audio Input Floats */ + + bmdDeckLinkConfigAnalogAudioInputScaleChannel1 = /* 'ais1' */ 0x61697331, + bmdDeckLinkConfigAnalogAudioInputScaleChannel2 = /* 'ais2' */ 0x61697332, + bmdDeckLinkConfigAnalogAudioInputScaleChannel3 = /* 'ais3' */ 0x61697333, + bmdDeckLinkConfigAnalogAudioInputScaleChannel4 = /* 'ais4' */ 0x61697334, + bmdDeckLinkConfigDigitalAudioInputScale = /* 'dais' */ 0x64616973, + bmdDeckLinkConfigMicrophoneInputGain = /* 'micg' */ 0x6D696367, + + /* Audio Output Integers */ + + bmdDeckLinkConfigAudioOutputAESAnalogSwitch = /* 'aoaa' */ 0x616F6161, + + /* Audio Output Floats */ + + bmdDeckLinkConfigAnalogAudioOutputScaleChannel1 = /* 'aos1' */ 0x616F7331, + bmdDeckLinkConfigAnalogAudioOutputScaleChannel2 = /* 'aos2' */ 0x616F7332, + bmdDeckLinkConfigAnalogAudioOutputScaleChannel3 = /* 'aos3' */ 0x616F7333, + bmdDeckLinkConfigAnalogAudioOutputScaleChannel4 = /* 'aos4' */ 0x616F7334, + bmdDeckLinkConfigDigitalAudioOutputScale = /* 'daos' */ 0x64616F73, + bmdDeckLinkConfigHeadphoneVolume = /* 'hvol' */ 0x68766F6C, + + /* Device Information Strings */ + + bmdDeckLinkConfigDeviceInformationLabel = /* 'dila' */ 0x64696C61, + bmdDeckLinkConfigDeviceInformationSerialNumber = /* 'disn' */ 0x6469736E, + bmdDeckLinkConfigDeviceInformationCompany = /* 'dico' */ 0x6469636F, + bmdDeckLinkConfigDeviceInformationPhone = /* 'diph' */ 0x64697068, + bmdDeckLinkConfigDeviceInformationEmail = /* 'diem' */ 0x6469656D, + bmdDeckLinkConfigDeviceInformationDate = /* 'dida' */ 0x64696461, + + /* Deck Control Integers */ + + bmdDeckLinkConfigDeckControlConnection = /* 'dcco' */ 0x6463636F +}; + +/* Enum BMDDeckLinkEncoderConfigurationID - DeckLink Encoder Configuration ID */ + +typedef uint32_t BMDDeckLinkEncoderConfigurationID; +enum _BMDDeckLinkEncoderConfigurationID { + + /* Video Encoder Integers */ + + bmdDeckLinkEncoderConfigPreferredBitDepth = /* 'epbr' */ 0x65706272, + bmdDeckLinkEncoderConfigFrameCodingMode = /* 'efcm' */ 0x6566636D, + + /* HEVC/H.265 Encoder Integers */ + + bmdDeckLinkEncoderConfigH265TargetBitrate = /* 'htbr' */ 0x68746272, + + /* DNxHR/DNxHD Compression ID */ + + bmdDeckLinkEncoderConfigDNxHRCompressionID = /* 'dcid' */ 0x64636964, + + /* DNxHR/DNxHD Level */ + + bmdDeckLinkEncoderConfigDNxHRLevel = /* 'dlev' */ 0x646C6576, + + /* Encoded Sample Decriptions */ + + bmdDeckLinkEncoderConfigMPEG4SampleDescription = /* 'stsE' */ 0x73747345, // Full MPEG4 sample description (aka SampleEntry of an 'stsd' atom-box). Useful for MediaFoundation, QuickTime, MKV and more + bmdDeckLinkEncoderConfigMPEG4CodecSpecificDesc = /* 'esds' */ 0x65736473 // Sample description extensions only (atom stream, each with size and fourCC header). Useful for AVFoundation, VideoToolbox, MKV and more +}; + +// Forward Declarations + +class IDeckLinkConfiguration; +class IDeckLinkEncoderConfiguration; + +/* Interface IDeckLinkConfiguration - DeckLink Configuration interface */ + +class IDeckLinkConfiguration : public IUnknown +{ +public: + virtual HRESULT SetFlag (/* in */ BMDDeckLinkConfigurationID cfgID, /* in */ bool value) = 0; + virtual HRESULT GetFlag (/* in */ BMDDeckLinkConfigurationID cfgID, /* out */ bool *value) = 0; + virtual HRESULT SetInt (/* in */ BMDDeckLinkConfigurationID cfgID, /* in */ int64_t value) = 0; + virtual HRESULT GetInt (/* in */ BMDDeckLinkConfigurationID cfgID, /* out */ int64_t *value) = 0; + virtual HRESULT SetFloat (/* in */ BMDDeckLinkConfigurationID cfgID, /* in */ double value) = 0; + virtual HRESULT GetFloat (/* in */ BMDDeckLinkConfigurationID cfgID, /* out */ double *value) = 0; + virtual HRESULT SetString (/* in */ BMDDeckLinkConfigurationID cfgID, /* in */ const char *value) = 0; + virtual HRESULT GetString (/* in */ BMDDeckLinkConfigurationID cfgID, /* out */ const char **value) = 0; + virtual HRESULT WriteConfigurationToPreferences (void) = 0; + +protected: + virtual ~IDeckLinkConfiguration () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkEncoderConfiguration - DeckLink Encoder Configuration interface. Obtained from IDeckLinkEncoderInput */ + +class IDeckLinkEncoderConfiguration : public IUnknown +{ +public: + virtual HRESULT SetFlag (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* in */ bool value) = 0; + virtual HRESULT GetFlag (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* out */ bool *value) = 0; + virtual HRESULT SetInt (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* in */ int64_t value) = 0; + virtual HRESULT GetInt (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* out */ int64_t *value) = 0; + virtual HRESULT SetFloat (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* in */ double value) = 0; + virtual HRESULT GetFloat (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* out */ double *value) = 0; + virtual HRESULT SetString (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* in */ const char *value) = 0; + virtual HRESULT GetString (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* out */ const char **value) = 0; + virtual HRESULT GetBytes (/* in */ BMDDeckLinkEncoderConfigurationID cfgID, /* out */ void *buffer /* optional */, /* in, out */ uint32_t *bufferSize) = 0; + +protected: + virtual ~IDeckLinkEncoderConfiguration () {} // call Release method to drop reference count +}; + +/* Functions */ + +extern "C" { + + +} + + +#endif /* defined(BMD_DECKLINKAPICONFIGURATION_H) */ diff --git a/nageru/decklink/DeckLinkAPIDeckControl.h b/nageru/decklink/DeckLinkAPIDeckControl.h new file mode 100644 index 0000000..1b76e10 --- /dev/null +++ b/nageru/decklink/DeckLinkAPIDeckControl.h @@ -0,0 +1,215 @@ +/* -LICENSE-START- +** Copyright (c) 2015 Blackmagic Design +** +** Permission is hereby granted, free of charge, to any person or organization +** obtaining a copy of the software and accompanying documentation covered by +** this license (the "Software") to use, reproduce, display, distribute, +** execute, and transmit the Software, and to prepare derivative works of the +** Software, and to permit third-parties to whom the Software is furnished to +** do so, all subject to the following: +** +** The copyright notices in the Software and this entire statement, including +** the above license grant, this restriction and the following disclaimer, +** must be included in all copies of the Software, in whole or in part, and +** all derivative works of the Software, unless such copies or derivative +** works are solely in the form of machine-executable object code generated by +** a source language processor. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +** -LICENSE-END- +*/ + +#ifndef BMD_DECKLINKAPIDECKCONTROL_H +#define BMD_DECKLINKAPIDECKCONTROL_H + + +#ifndef BMD_CONST + #if defined(_MSC_VER) + #define BMD_CONST __declspec(selectany) static const + #else + #define BMD_CONST static const + #endif +#endif + +// Type Declarations + + +// Interface ID Declarations + +BMD_CONST REFIID IID_IDeckLinkDeckControlStatusCallback = /* 53436FFB-B434-4906-BADC-AE3060FFE8EF */ {0x53,0x43,0x6F,0xFB,0xB4,0x34,0x49,0x06,0xBA,0xDC,0xAE,0x30,0x60,0xFF,0xE8,0xEF}; +BMD_CONST REFIID IID_IDeckLinkDeckControl = /* 8E1C3ACE-19C7-4E00-8B92-D80431D958BE */ {0x8E,0x1C,0x3A,0xCE,0x19,0xC7,0x4E,0x00,0x8B,0x92,0xD8,0x04,0x31,0xD9,0x58,0xBE}; + +/* Enum BMDDeckControlMode - DeckControl mode */ + +typedef uint32_t BMDDeckControlMode; +enum _BMDDeckControlMode { + bmdDeckControlNotOpened = /* 'ntop' */ 0x6E746F70, + bmdDeckControlVTRControlMode = /* 'vtrc' */ 0x76747263, + bmdDeckControlExportMode = /* 'expm' */ 0x6578706D, + bmdDeckControlCaptureMode = /* 'capm' */ 0x6361706D +}; + +/* Enum BMDDeckControlEvent - DeckControl event */ + +typedef uint32_t BMDDeckControlEvent; +enum _BMDDeckControlEvent { + bmdDeckControlAbortedEvent = /* 'abte' */ 0x61627465, // This event is triggered when a capture or edit-to-tape operation is aborted. + + /* Export-To-Tape events */ + + bmdDeckControlPrepareForExportEvent = /* 'pfee' */ 0x70666565, // This event is triggered a few frames before reaching the in-point. IDeckLinkInput::StartScheduledPlayback() should be called at this point. + bmdDeckControlExportCompleteEvent = /* 'exce' */ 0x65786365, // This event is triggered a few frames after reaching the out-point. At this point, it is safe to stop playback. + + /* Capture events */ + + bmdDeckControlPrepareForCaptureEvent = /* 'pfce' */ 0x70666365, // This event is triggered a few frames before reaching the in-point. The serial timecode attached to IDeckLinkVideoInputFrames is now valid. + bmdDeckControlCaptureCompleteEvent = /* 'ccev' */ 0x63636576 // This event is triggered a few frames after reaching the out-point. +}; + +/* Enum BMDDeckControlVTRControlState - VTR Control state */ + +typedef uint32_t BMDDeckControlVTRControlState; +enum _BMDDeckControlVTRControlState { + bmdDeckControlNotInVTRControlMode = /* 'nvcm' */ 0x6E76636D, + bmdDeckControlVTRControlPlaying = /* 'vtrp' */ 0x76747270, + bmdDeckControlVTRControlRecording = /* 'vtrr' */ 0x76747272, + bmdDeckControlVTRControlStill = /* 'vtra' */ 0x76747261, + bmdDeckControlVTRControlShuttleForward = /* 'vtsf' */ 0x76747366, + bmdDeckControlVTRControlShuttleReverse = /* 'vtsr' */ 0x76747372, + bmdDeckControlVTRControlJogForward = /* 'vtjf' */ 0x76746A66, + bmdDeckControlVTRControlJogReverse = /* 'vtjr' */ 0x76746A72, + bmdDeckControlVTRControlStopped = /* 'vtro' */ 0x7674726F +}; + +/* Enum BMDDeckControlStatusFlags - Deck Control status flags */ + +typedef uint32_t BMDDeckControlStatusFlags; +enum _BMDDeckControlStatusFlags { + bmdDeckControlStatusDeckConnected = 1 << 0, + bmdDeckControlStatusRemoteMode = 1 << 1, + bmdDeckControlStatusRecordInhibited = 1 << 2, + bmdDeckControlStatusCassetteOut = 1 << 3 +}; + +/* Enum BMDDeckControlExportModeOpsFlags - Export mode flags */ + +typedef uint32_t BMDDeckControlExportModeOpsFlags; +enum _BMDDeckControlExportModeOpsFlags { + bmdDeckControlExportModeInsertVideo = 1 << 0, + bmdDeckControlExportModeInsertAudio1 = 1 << 1, + bmdDeckControlExportModeInsertAudio2 = 1 << 2, + bmdDeckControlExportModeInsertAudio3 = 1 << 3, + bmdDeckControlExportModeInsertAudio4 = 1 << 4, + bmdDeckControlExportModeInsertAudio5 = 1 << 5, + bmdDeckControlExportModeInsertAudio6 = 1 << 6, + bmdDeckControlExportModeInsertAudio7 = 1 << 7, + bmdDeckControlExportModeInsertAudio8 = 1 << 8, + bmdDeckControlExportModeInsertAudio9 = 1 << 9, + bmdDeckControlExportModeInsertAudio10 = 1 << 10, + bmdDeckControlExportModeInsertAudio11 = 1 << 11, + bmdDeckControlExportModeInsertAudio12 = 1 << 12, + bmdDeckControlExportModeInsertTimeCode = 1 << 13, + bmdDeckControlExportModeInsertAssemble = 1 << 14, + bmdDeckControlExportModeInsertPreview = 1 << 15, + bmdDeckControlUseManualExport = 1 << 16 +}; + +/* Enum BMDDeckControlError - Deck Control error */ + +typedef uint32_t BMDDeckControlError; +enum _BMDDeckControlError { + bmdDeckControlNoError = /* 'noer' */ 0x6E6F6572, + bmdDeckControlModeError = /* 'moer' */ 0x6D6F6572, + bmdDeckControlMissedInPointError = /* 'mier' */ 0x6D696572, + bmdDeckControlDeckTimeoutError = /* 'dter' */ 0x64746572, + bmdDeckControlCommandFailedError = /* 'cfer' */ 0x63666572, + bmdDeckControlDeviceAlreadyOpenedError = /* 'dalo' */ 0x64616C6F, + bmdDeckControlFailedToOpenDeviceError = /* 'fder' */ 0x66646572, + bmdDeckControlInLocalModeError = /* 'lmer' */ 0x6C6D6572, + bmdDeckControlEndOfTapeError = /* 'eter' */ 0x65746572, + bmdDeckControlUserAbortError = /* 'uaer' */ 0x75616572, + bmdDeckControlNoTapeInDeckError = /* 'nter' */ 0x6E746572, + bmdDeckControlNoVideoFromCardError = /* 'nvfc' */ 0x6E766663, + bmdDeckControlNoCommunicationError = /* 'ncom' */ 0x6E636F6D, + bmdDeckControlBufferTooSmallError = /* 'btsm' */ 0x6274736D, + bmdDeckControlBadChecksumError = /* 'chks' */ 0x63686B73, + bmdDeckControlUnknownError = /* 'uner' */ 0x756E6572 +}; + +// Forward Declarations + +class IDeckLinkDeckControlStatusCallback; +class IDeckLinkDeckControl; + +/* Interface IDeckLinkDeckControlStatusCallback - Deck control state change callback. */ + +class IDeckLinkDeckControlStatusCallback : public IUnknown +{ +public: + virtual HRESULT TimecodeUpdate (/* in */ BMDTimecodeBCD currentTimecode) = 0; + virtual HRESULT VTRControlStateChanged (/* in */ BMDDeckControlVTRControlState newState, /* in */ BMDDeckControlError error) = 0; + virtual HRESULT DeckControlEventReceived (/* in */ BMDDeckControlEvent event, /* in */ BMDDeckControlError error) = 0; + virtual HRESULT DeckControlStatusChanged (/* in */ BMDDeckControlStatusFlags flags, /* in */ uint32_t mask) = 0; + +protected: + virtual ~IDeckLinkDeckControlStatusCallback () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkDeckControl - Deck Control main interface */ + +class IDeckLinkDeckControl : public IUnknown +{ +public: + virtual HRESULT Open (/* in */ BMDTimeScale timeScale, /* in */ BMDTimeValue timeValue, /* in */ bool timecodeIsDropFrame, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT Close (/* in */ bool standbyOn) = 0; + virtual HRESULT GetCurrentState (/* out */ BMDDeckControlMode *mode, /* out */ BMDDeckControlVTRControlState *vtrControlState, /* out */ BMDDeckControlStatusFlags *flags) = 0; + virtual HRESULT SetStandby (/* in */ bool standbyOn) = 0; + virtual HRESULT SendCommand (/* in */ uint8_t *inBuffer, /* in */ uint32_t inBufferSize, /* out */ uint8_t *outBuffer, /* out */ uint32_t *outDataSize, /* in */ uint32_t outBufferSize, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT Play (/* out */ BMDDeckControlError *error) = 0; + virtual HRESULT Stop (/* out */ BMDDeckControlError *error) = 0; + virtual HRESULT TogglePlayStop (/* out */ BMDDeckControlError *error) = 0; + virtual HRESULT Eject (/* out */ BMDDeckControlError *error) = 0; + virtual HRESULT GoToTimecode (/* in */ BMDTimecodeBCD timecode, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT FastForward (/* in */ bool viewTape, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT Rewind (/* in */ bool viewTape, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT StepForward (/* out */ BMDDeckControlError *error) = 0; + virtual HRESULT StepBack (/* out */ BMDDeckControlError *error) = 0; + virtual HRESULT Jog (/* in */ double rate, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT Shuttle (/* in */ double rate, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT GetTimecodeString (/* out */ const char **currentTimeCode, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT GetTimecode (/* out */ IDeckLinkTimecode **currentTimecode, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT GetTimecodeBCD (/* out */ BMDTimecodeBCD *currentTimecode, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT SetPreroll (/* in */ uint32_t prerollSeconds) = 0; + virtual HRESULT GetPreroll (/* out */ uint32_t *prerollSeconds) = 0; + virtual HRESULT SetExportOffset (/* in */ int32_t exportOffsetFields) = 0; + virtual HRESULT GetExportOffset (/* out */ int32_t *exportOffsetFields) = 0; + virtual HRESULT GetManualExportOffset (/* out */ int32_t *deckManualExportOffsetFields) = 0; + virtual HRESULT SetCaptureOffset (/* in */ int32_t captureOffsetFields) = 0; + virtual HRESULT GetCaptureOffset (/* out */ int32_t *captureOffsetFields) = 0; + virtual HRESULT StartExport (/* in */ BMDTimecodeBCD inTimecode, /* in */ BMDTimecodeBCD outTimecode, /* in */ BMDDeckControlExportModeOpsFlags exportModeOps, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT StartCapture (/* in */ bool useVITC, /* in */ BMDTimecodeBCD inTimecode, /* in */ BMDTimecodeBCD outTimecode, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT GetDeviceID (/* out */ uint16_t *deviceId, /* out */ BMDDeckControlError *error) = 0; + virtual HRESULT Abort (void) = 0; + virtual HRESULT CrashRecordStart (/* out */ BMDDeckControlError *error) = 0; + virtual HRESULT CrashRecordStop (/* out */ BMDDeckControlError *error) = 0; + virtual HRESULT SetCallback (/* in */ IDeckLinkDeckControlStatusCallback *callback) = 0; + +protected: + virtual ~IDeckLinkDeckControl () {} // call Release method to drop reference count +}; + +/* Functions */ + +extern "C" { + + +} + + +#endif /* defined(BMD_DECKLINKAPIDECKCONTROL_H) */ diff --git a/nageru/decklink/DeckLinkAPIDiscovery.h b/nageru/decklink/DeckLinkAPIDiscovery.h new file mode 100644 index 0000000..93ca66b --- /dev/null +++ b/nageru/decklink/DeckLinkAPIDiscovery.h @@ -0,0 +1,71 @@ +/* -LICENSE-START- +** Copyright (c) 2015 Blackmagic Design +** +** Permission is hereby granted, free of charge, to any person or organization +** obtaining a copy of the software and accompanying documentation covered by +** this license (the "Software") to use, reproduce, display, distribute, +** execute, and transmit the Software, and to prepare derivative works of the +** Software, and to permit third-parties to whom the Software is furnished to +** do so, all subject to the following: +** +** The copyright notices in the Software and this entire statement, including +** the above license grant, this restriction and the following disclaimer, +** must be included in all copies of the Software, in whole or in part, and +** all derivative works of the Software, unless such copies or derivative +** works are solely in the form of machine-executable object code generated by +** a source language processor. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +** -LICENSE-END- +*/ + +#ifndef BMD_DECKLINKAPIDISCOVERY_H +#define BMD_DECKLINKAPIDISCOVERY_H + + +#ifndef BMD_CONST + #if defined(_MSC_VER) + #define BMD_CONST __declspec(selectany) static const + #else + #define BMD_CONST static const + #endif +#endif + +// Type Declarations + + +// Interface ID Declarations + +BMD_CONST REFIID IID_IDeckLink = /* C418FBDD-0587-48ED-8FE5-640F0A14AF91 */ {0xC4,0x18,0xFB,0xDD,0x05,0x87,0x48,0xED,0x8F,0xE5,0x64,0x0F,0x0A,0x14,0xAF,0x91}; + +// Forward Declarations + +class IDeckLink; + +/* Interface IDeckLink - represents a DeckLink device */ + +class IDeckLink : public IUnknown +{ +public: + virtual HRESULT GetModelName (/* out */ const char **modelName) = 0; + virtual HRESULT GetDisplayName (/* out */ const char **displayName) = 0; + +protected: + virtual ~IDeckLink () {} // call Release method to drop reference count +}; + +/* Functions */ + +extern "C" { + + +} + + +#endif /* defined(BMD_DECKLINKAPIDISCOVERY_H) */ diff --git a/nageru/decklink/DeckLinkAPIDispatch.cpp b/nageru/decklink/DeckLinkAPIDispatch.cpp new file mode 100755 index 0000000..a3d2f2b --- /dev/null +++ b/nageru/decklink/DeckLinkAPIDispatch.cpp @@ -0,0 +1,146 @@ +/* -LICENSE-START- +** Copyright (c) 2009 Blackmagic Design +** +** Permission is hereby granted, free of charge, to any person or organization +** obtaining a copy of the software and accompanying documentation covered by +** this license (the "Software") to use, reproduce, display, distribute, +** execute, and transmit the Software, and to prepare derivative works of the +** Software, and to permit third-parties to whom the Software is furnished to +** do so, all subject to the following: +** +** The copyright notices in the Software and this entire statement, including +** the above license grant, this restriction and the following disclaimer, +** must be included in all copies of the Software, in whole or in part, and +** all derivative works of the Software, unless such copies or derivative +** works are solely in the form of machine-executable object code generated by +** a source language processor. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +** -LICENSE-END- +**/ + +#include +#include +#include + +#include "DeckLinkAPI.h" + +#define kDeckLinkAPI_Name "libDeckLinkAPI.so" +#define KDeckLinkPreviewAPI_Name "libDeckLinkPreviewAPI.so" + +typedef IDeckLinkIterator* (*CreateIteratorFunc)(void); +typedef IDeckLinkAPIInformation* (*CreateAPIInformationFunc)(void); +typedef IDeckLinkGLScreenPreviewHelper* (*CreateOpenGLScreenPreviewHelperFunc)(void); +typedef IDeckLinkVideoConversion* (*CreateVideoConversionInstanceFunc)(void); +typedef IDeckLinkDiscovery* (*CreateDeckLinkDiscoveryInstanceFunc)(void); + +static pthread_once_t gDeckLinkOnceControl = PTHREAD_ONCE_INIT; +static pthread_once_t gPreviewOnceControl = PTHREAD_ONCE_INIT; + +static bool gLoadedDeckLinkAPI = false; + +static CreateIteratorFunc gCreateIteratorFunc = NULL; +static CreateAPIInformationFunc gCreateAPIInformationFunc = NULL; +static CreateOpenGLScreenPreviewHelperFunc gCreateOpenGLPreviewFunc = NULL; +static CreateVideoConversionInstanceFunc gCreateVideoConversionFunc = NULL; +static CreateDeckLinkDiscoveryInstanceFunc gCreateDeckLinkDiscoveryFunc = NULL; + +void InitDeckLinkAPI (void) +{ + void *libraryHandle; + + libraryHandle = dlopen(kDeckLinkAPI_Name, RTLD_NOW|RTLD_GLOBAL); + if (!libraryHandle) + { + fprintf(stderr, "%s\n", dlerror()); + return; + } + + gLoadedDeckLinkAPI = true; + + gCreateIteratorFunc = (CreateIteratorFunc)dlsym(libraryHandle, "CreateDeckLinkIteratorInstance_0002"); + if (!gCreateIteratorFunc) + fprintf(stderr, "%s\n", dlerror()); + gCreateAPIInformationFunc = (CreateAPIInformationFunc)dlsym(libraryHandle, "CreateDeckLinkAPIInformationInstance_0001"); + if (!gCreateAPIInformationFunc) + fprintf(stderr, "%s\n", dlerror()); + gCreateVideoConversionFunc = (CreateVideoConversionInstanceFunc)dlsym(libraryHandle, "CreateVideoConversionInstance_0001"); + if (!gCreateVideoConversionFunc) + fprintf(stderr, "%s\n", dlerror()); + gCreateDeckLinkDiscoveryFunc = (CreateDeckLinkDiscoveryInstanceFunc)dlsym(libraryHandle, "CreateDeckLinkDiscoveryInstance_0001"); + if (!gCreateDeckLinkDiscoveryFunc) + fprintf(stderr, "%s\n", dlerror()); +} + +void InitDeckLinkPreviewAPI (void) +{ + void *libraryHandle; + + libraryHandle = dlopen(KDeckLinkPreviewAPI_Name, RTLD_NOW|RTLD_GLOBAL); + if (!libraryHandle) + { + fprintf(stderr, "%s\n", dlerror()); + return; + } + gCreateOpenGLPreviewFunc = (CreateOpenGLScreenPreviewHelperFunc)dlsym(libraryHandle, "CreateOpenGLScreenPreviewHelper_0001"); + if (!gCreateOpenGLPreviewFunc) + fprintf(stderr, "%s\n", dlerror()); +} + +bool IsDeckLinkAPIPresent (void) +{ + // If the DeckLink API dynamic library was successfully loaded, return this knowledge to the caller + return gLoadedDeckLinkAPI; +} + +IDeckLinkIterator* CreateDeckLinkIteratorInstance (void) +{ + pthread_once(&gDeckLinkOnceControl, InitDeckLinkAPI); + + if (gCreateIteratorFunc == NULL) + return NULL; + return gCreateIteratorFunc(); +} + +IDeckLinkAPIInformation* CreateDeckLinkAPIInformationInstance (void) +{ + pthread_once(&gDeckLinkOnceControl, InitDeckLinkAPI); + + if (gCreateAPIInformationFunc == NULL) + return NULL; + return gCreateAPIInformationFunc(); +} + +IDeckLinkGLScreenPreviewHelper* CreateOpenGLScreenPreviewHelper (void) +{ + pthread_once(&gDeckLinkOnceControl, InitDeckLinkAPI); + pthread_once(&gPreviewOnceControl, InitDeckLinkPreviewAPI); + + if (gCreateOpenGLPreviewFunc == NULL) + return NULL; + return gCreateOpenGLPreviewFunc(); +} + +IDeckLinkVideoConversion* CreateVideoConversionInstance (void) +{ + pthread_once(&gDeckLinkOnceControl, InitDeckLinkAPI); + + if (gCreateVideoConversionFunc == NULL) + return NULL; + return gCreateVideoConversionFunc(); +} + +IDeckLinkDiscovery* CreateDeckLinkDiscoveryInstance (void) +{ + pthread_once(&gDeckLinkOnceControl, InitDeckLinkAPI); + + if (gCreateDeckLinkDiscoveryFunc == NULL) + return NULL; + return gCreateDeckLinkDiscoveryFunc(); +} diff --git a/nageru/decklink/DeckLinkAPIModes.h b/nageru/decklink/DeckLinkAPIModes.h new file mode 100644 index 0000000..c508af7 --- /dev/null +++ b/nageru/decklink/DeckLinkAPIModes.h @@ -0,0 +1,192 @@ +/* -LICENSE-START- +** Copyright (c) 2015 Blackmagic Design +** +** Permission is hereby granted, free of charge, to any person or organization +** obtaining a copy of the software and accompanying documentation covered by +** this license (the "Software") to use, reproduce, display, distribute, +** execute, and transmit the Software, and to prepare derivative works of the +** Software, and to permit third-parties to whom the Software is furnished to +** do so, all subject to the following: +** +** The copyright notices in the Software and this entire statement, including +** the above license grant, this restriction and the following disclaimer, +** must be included in all copies of the Software, in whole or in part, and +** all derivative works of the Software, unless such copies or derivative +** works are solely in the form of machine-executable object code generated by +** a source language processor. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +** -LICENSE-END- +*/ + +#ifndef BMD_DECKLINKAPIMODES_H +#define BMD_DECKLINKAPIMODES_H + + +#ifndef BMD_CONST + #if defined(_MSC_VER) + #define BMD_CONST __declspec(selectany) static const + #else + #define BMD_CONST static const + #endif +#endif + +// Type Declarations + + +// Interface ID Declarations + +BMD_CONST REFIID IID_IDeckLinkDisplayModeIterator = /* 9C88499F-F601-4021-B80B-032E4EB41C35 */ {0x9C,0x88,0x49,0x9F,0xF6,0x01,0x40,0x21,0xB8,0x0B,0x03,0x2E,0x4E,0xB4,0x1C,0x35}; +BMD_CONST REFIID IID_IDeckLinkDisplayMode = /* 3EB2C1AB-0A3D-4523-A3AD-F40D7FB14E78 */ {0x3E,0xB2,0xC1,0xAB,0x0A,0x3D,0x45,0x23,0xA3,0xAD,0xF4,0x0D,0x7F,0xB1,0x4E,0x78}; + +/* Enum BMDDisplayMode - Video display modes */ + +typedef uint32_t BMDDisplayMode; +enum _BMDDisplayMode { + + /* SD Modes */ + + bmdModeNTSC = /* 'ntsc' */ 0x6E747363, + bmdModeNTSC2398 = /* 'nt23' */ 0x6E743233, // 3:2 pulldown + bmdModePAL = /* 'pal ' */ 0x70616C20, + bmdModeNTSCp = /* 'ntsp' */ 0x6E747370, + bmdModePALp = /* 'palp' */ 0x70616C70, + + /* HD 1080 Modes */ + + bmdModeHD1080p2398 = /* '23ps' */ 0x32337073, + bmdModeHD1080p24 = /* '24ps' */ 0x32347073, + bmdModeHD1080p25 = /* 'Hp25' */ 0x48703235, + bmdModeHD1080p2997 = /* 'Hp29' */ 0x48703239, + bmdModeHD1080p30 = /* 'Hp30' */ 0x48703330, + bmdModeHD1080i50 = /* 'Hi50' */ 0x48693530, + bmdModeHD1080i5994 = /* 'Hi59' */ 0x48693539, + bmdModeHD1080i6000 = /* 'Hi60' */ 0x48693630, // N.B. This _really_ is 60.00 Hz. + bmdModeHD1080p50 = /* 'Hp50' */ 0x48703530, + bmdModeHD1080p5994 = /* 'Hp59' */ 0x48703539, + bmdModeHD1080p6000 = /* 'Hp60' */ 0x48703630, // N.B. This _really_ is 60.00 Hz. + + /* HD 720 Modes */ + + bmdModeHD720p50 = /* 'hp50' */ 0x68703530, + bmdModeHD720p5994 = /* 'hp59' */ 0x68703539, + bmdModeHD720p60 = /* 'hp60' */ 0x68703630, + + /* 2k Modes */ + + bmdMode2k2398 = /* '2k23' */ 0x326B3233, + bmdMode2k24 = /* '2k24' */ 0x326B3234, + bmdMode2k25 = /* '2k25' */ 0x326B3235, + + /* DCI Modes (output only) */ + + bmdMode2kDCI2398 = /* '2d23' */ 0x32643233, + bmdMode2kDCI24 = /* '2d24' */ 0x32643234, + bmdMode2kDCI25 = /* '2d25' */ 0x32643235, + + /* 4k Modes */ + + bmdMode4K2160p2398 = /* '4k23' */ 0x346B3233, + bmdMode4K2160p24 = /* '4k24' */ 0x346B3234, + bmdMode4K2160p25 = /* '4k25' */ 0x346B3235, + bmdMode4K2160p2997 = /* '4k29' */ 0x346B3239, + bmdMode4K2160p30 = /* '4k30' */ 0x346B3330, + bmdMode4K2160p50 = /* '4k50' */ 0x346B3530, + bmdMode4K2160p5994 = /* '4k59' */ 0x346B3539, + bmdMode4K2160p60 = /* '4k60' */ 0x346B3630, + + /* DCI Modes (output only) */ + + bmdMode4kDCI2398 = /* '4d23' */ 0x34643233, + bmdMode4kDCI24 = /* '4d24' */ 0x34643234, + bmdMode4kDCI25 = /* '4d25' */ 0x34643235, + + /* Special Modes */ + + bmdModeUnknown = /* 'iunk' */ 0x69756E6B +}; + +/* Enum BMDFieldDominance - Video field dominance */ + +typedef uint32_t BMDFieldDominance; +enum _BMDFieldDominance { + bmdUnknownFieldDominance = 0, + bmdLowerFieldFirst = /* 'lowr' */ 0x6C6F7772, + bmdUpperFieldFirst = /* 'uppr' */ 0x75707072, + bmdProgressiveFrame = /* 'prog' */ 0x70726F67, + bmdProgressiveSegmentedFrame = /* 'psf ' */ 0x70736620 +}; + +/* Enum BMDPixelFormat - Video pixel formats supported for output/input */ + +typedef uint32_t BMDPixelFormat; +enum _BMDPixelFormat { + bmdFormat8BitYUV = /* '2vuy' */ 0x32767579, + bmdFormat10BitYUV = /* 'v210' */ 0x76323130, + bmdFormat8BitARGB = 32, + bmdFormat8BitBGRA = /* 'BGRA' */ 0x42475241, + bmdFormat10BitRGB = /* 'r210' */ 0x72323130, // Big-endian RGB 10-bit per component with SMPTE video levels (64-960). Packed as 2:10:10:10 + bmdFormat12BitRGB = /* 'R12B' */ 0x52313242, // Big-endian RGB 12-bit per component with full range (0-4095). Packed as 12-bit per component + bmdFormat12BitRGBLE = /* 'R12L' */ 0x5231324C, // Little-endian RGB 12-bit per component with full range (0-4095). Packed as 12-bit per component + bmdFormat10BitRGBXLE = /* 'R10l' */ 0x5231306C, // Little-endian 10-bit RGB with SMPTE video levels (64-940) + bmdFormat10BitRGBX = /* 'R10b' */ 0x52313062, // Big-endian 10-bit RGB with SMPTE video levels (64-940) + bmdFormatH265 = /* 'hev1' */ 0x68657631 // High Efficiency Video Coding (HEVC/h.265) +}; + +/* Enum BMDDisplayModeFlags - Flags to describe the characteristics of an IDeckLinkDisplayMode. */ + +typedef uint32_t BMDDisplayModeFlags; +enum _BMDDisplayModeFlags { + bmdDisplayModeSupports3D = 1 << 0, + bmdDisplayModeColorspaceRec601 = 1 << 1, + bmdDisplayModeColorspaceRec709 = 1 << 2 +}; + +// Forward Declarations + +class IDeckLinkDisplayModeIterator; +class IDeckLinkDisplayMode; + +/* Interface IDeckLinkDisplayModeIterator - enumerates over supported input/output display modes. */ + +class IDeckLinkDisplayModeIterator : public IUnknown +{ +public: + virtual HRESULT Next (/* out */ IDeckLinkDisplayMode **deckLinkDisplayMode) = 0; + +protected: + virtual ~IDeckLinkDisplayModeIterator () {} // call Release method to drop reference count +}; + +/* Interface IDeckLinkDisplayMode - represents a display mode */ + +class IDeckLinkDisplayMode : public IUnknown +{ +public: + virtual HRESULT GetName (/* out */ const char **name) = 0; + virtual BMDDisplayMode GetDisplayMode (void) = 0; + virtual long GetWidth (void) = 0; + virtual long GetHeight (void) = 0; + virtual HRESULT GetFrameRate (/* out */ BMDTimeValue *frameDuration, /* out */ BMDTimeScale *timeScale) = 0; + virtual BMDFieldDominance GetFieldDominance (void) = 0; + virtual BMDDisplayModeFlags GetFlags (void) = 0; + +protected: + virtual ~IDeckLinkDisplayMode () {} // call Release method to drop reference count +}; + +/* Functions */ + +extern "C" { + + +} + + +#endif /* defined(BMD_DECKLINKAPIMODES_H) */ diff --git a/nageru/decklink/DeckLinkAPITypes.h b/nageru/decklink/DeckLinkAPITypes.h new file mode 100644 index 0000000..bc6d581 --- /dev/null +++ b/nageru/decklink/DeckLinkAPITypes.h @@ -0,0 +1,120 @@ +/* -LICENSE-START- +** Copyright (c) 2015 Blackmagic Design +** +** Permission is hereby granted, free of charge, to any person or organization +** obtaining a copy of the software and accompanying documentation covered by +** this license (the "Software") to use, reproduce, display, distribute, +** execute, and transmit the Software, and to prepare derivative works of the +** Software, and to permit third-parties to whom the Software is furnished to +** do so, all subject to the following: +** +** The copyright notices in the Software and this entire statement, including +** the above license grant, this restriction and the following disclaimer, +** must be included in all copies of the Software, in whole or in part, and +** all derivative works of the Software, unless such copies or derivative +** works are solely in the form of machine-executable object code generated by +** a source language processor. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +** -LICENSE-END- +*/ + +#ifndef BMD_DECKLINKAPITYPES_H +#define BMD_DECKLINKAPITYPES_H + + +#ifndef BMD_CONST + #if defined(_MSC_VER) + #define BMD_CONST __declspec(selectany) static const + #else + #define BMD_CONST static const + #endif +#endif + +// Type Declarations + +typedef int64_t BMDTimeValue; +typedef int64_t BMDTimeScale; +typedef uint32_t BMDTimecodeBCD; +typedef uint32_t BMDTimecodeUserBits; + +// Interface ID Declarations + +BMD_CONST REFIID IID_IDeckLinkTimecode = /* BC6CFBD3-8317-4325-AC1C-1216391E9340 */ {0xBC,0x6C,0xFB,0xD3,0x83,0x17,0x43,0x25,0xAC,0x1C,0x12,0x16,0x39,0x1E,0x93,0x40}; + +/* Enum BMDTimecodeFlags - Timecode flags */ + +typedef uint32_t BMDTimecodeFlags; +enum _BMDTimecodeFlags { + bmdTimecodeFlagDefault = 0, + bmdTimecodeIsDropFrame = 1 << 0, + bmdTimecodeFieldMark = 1 << 1 +}; + +/* Enum BMDVideoConnection - Video connection types */ + +typedef uint32_t BMDVideoConnection; +enum _BMDVideoConnection { + bmdVideoConnectionSDI = 1 << 0, + bmdVideoConnectionHDMI = 1 << 1, + bmdVideoConnectionOpticalSDI = 1 << 2, + bmdVideoConnectionComponent = 1 << 3, + bmdVideoConnectionComposite = 1 << 4, + bmdVideoConnectionSVideo = 1 << 5 +}; + +/* Enum BMDAudioConnection - Audio connection types */ + +typedef uint32_t BMDAudioConnection; +enum _BMDAudioConnection { + bmdAudioConnectionEmbedded = 1 << 0, + bmdAudioConnectionAESEBU = 1 << 1, + bmdAudioConnectionAnalog = 1 << 2, + bmdAudioConnectionAnalogXLR = 1 << 3, + bmdAudioConnectionAnalogRCA = 1 << 4, + bmdAudioConnectionMicrophone = 1 << 5, + bmdAudioConnectionHeadphones = 1 << 6 +}; + +/* Enum BMDDeckControlConnection - Deck control connections */ + +typedef uint32_t BMDDeckControlConnection; +enum _BMDDeckControlConnection { + bmdDeckControlConnectionRS422Remote1 = 1 << 0, + bmdDeckControlConnectionRS422Remote2 = 1 << 1 +}; + +// Forward Declarations + +class IDeckLinkTimecode; + +/* Interface IDeckLinkTimecode - Used for video frame timecode representation. */ + +class IDeckLinkTimecode : public IUnknown +{ +public: + virtual BMDTimecodeBCD GetBCD (void) = 0; + virtual HRESULT GetComponents (/* out */ uint8_t *hours, /* out */ uint8_t *minutes, /* out */ uint8_t *seconds, /* out */ uint8_t *frames) = 0; + virtual HRESULT GetString (/* out */ const char **timecode) = 0; + virtual BMDTimecodeFlags GetFlags (void) = 0; + virtual HRESULT GetTimecodeUserBits (/* out */ BMDTimecodeUserBits *userBits) = 0; + +protected: + virtual ~IDeckLinkTimecode () {} // call Release method to drop reference count +}; + +/* Functions */ + +extern "C" { + + +} + + +#endif /* defined(BMD_DECKLINKAPITYPES_H) */ diff --git a/nageru/decklink/LinuxCOM.h b/nageru/decklink/LinuxCOM.h new file mode 100644 index 0000000..2b13697 --- /dev/null +++ b/nageru/decklink/LinuxCOM.h @@ -0,0 +1,99 @@ +/* -LICENSE-START- +** Copyright (c) 2009 Blackmagic Design +** +** Permission is hereby granted, free of charge, to any person or organization +** obtaining a copy of the software and accompanying documentation covered by +** this license (the "Software") to use, reproduce, display, distribute, +** execute, and transmit the Software, and to prepare derivative works of the +** Software, and to permit third-parties to whom the Software is furnished to +** do so, all subject to the following: +** +** The copyright notices in the Software and this entire statement, including +** the above license grant, this restriction and the following disclaimer, +** must be included in all copies of the Software, in whole or in part, and +** all derivative works of the Software, unless such copies or derivative +** works are solely in the form of machine-executable object code generated by +** a source language processor. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +** -LICENSE-END- +*/ + +#ifndef __LINUX_COM_H_ +#define __LINUX_COM_H_ + +struct REFIID +{ + unsigned char byte0; + unsigned char byte1; + unsigned char byte2; + unsigned char byte3; + unsigned char byte4; + unsigned char byte5; + unsigned char byte6; + unsigned char byte7; + unsigned char byte8; + unsigned char byte9; + unsigned char byte10; + unsigned char byte11; + unsigned char byte12; + unsigned char byte13; + unsigned char byte14; + unsigned char byte15; +}; + +typedef REFIID CFUUIDBytes; +#define CFUUIDGetUUIDBytes(x) x + +typedef int HRESULT; +typedef unsigned long ULONG; +typedef void *LPVOID; + +#define SUCCEEDED(Status) ((HRESULT)(Status) >= 0) +#define FAILED(Status) ((HRESULT)(Status)<0) + +#define IS_ERROR(Status) ((unsigned long)(Status) >> 31 == SEVERITY_ERROR) +#define HRESULT_CODE(hr) ((hr) & 0xFFFF) +#define HRESULT_FACILITY(hr) (((hr) >> 16) & 0x1fff) +#define HRESULT_SEVERITY(hr) (((hr) >> 31) & 0x1) +#define SEVERITY_SUCCESS 0 +#define SEVERITY_ERROR 1 + +#define MAKE_HRESULT(sev,fac,code) ((HRESULT) (((unsigned long)(sev)<<31) | ((unsigned long)(fac)<<16) | ((unsigned long)(code))) ) + +#define S_OK ((HRESULT)0x00000000L) +#define S_FALSE ((HRESULT)0x00000001L) +#define E_UNEXPECTED ((HRESULT)0x8000FFFFL) +#define E_NOTIMPL ((HRESULT)0x80000001L) +#define E_OUTOFMEMORY ((HRESULT)0x80000002L) +#define E_INVALIDARG ((HRESULT)0x80000003L) +#define E_NOINTERFACE ((HRESULT)0x80000004L) +#define E_POINTER ((HRESULT)0x80000005L) +#define E_HANDLE ((HRESULT)0x80000006L) +#define E_ABORT ((HRESULT)0x80000007L) +#define E_FAIL ((HRESULT)0x80000008L) +#define E_ACCESSDENIED ((HRESULT)0x80000009L) + +#define STDMETHODCALLTYPE + +#define IID_IUnknown (REFIID){0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xC0,0x00,0x00,0x00,0x00,0x00,0x00,0x46} +#define IUnknownUUID IID_IUnknown + +#ifdef __cplusplus +class IUnknown +{ + public: + virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv) = 0; + virtual ULONG STDMETHODCALLTYPE AddRef(void) = 0; + virtual ULONG STDMETHODCALLTYPE Release(void) = 0; +}; +#endif + +#endif + diff --git a/nageru/decklink_capture.cpp b/nageru/decklink_capture.cpp new file mode 100644 index 0000000..881e181 --- /dev/null +++ b/nageru/decklink_capture.cpp @@ -0,0 +1,436 @@ +#include "decklink_capture.h" + +#include +#include +#include +#include +#include +#ifdef __SSE2__ +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bmusb/bmusb.h" +#include "decklink_util.h" +#include "flags.h" +#include "memcpy_interleaved.h" +#include "v210_converter.h" + +#define FRAME_SIZE (8 << 20) // 8 MB. + +using namespace std; +using namespace std::chrono; +using namespace std::placeholders; +using namespace bmusb; + +namespace { + +BMDPixelFormat pixel_format_to_bmd(PixelFormat pixel_format) +{ + switch (pixel_format) { + case PixelFormat_8BitYCbCr: + return bmdFormat8BitYUV; + case PixelFormat_10BitYCbCr: + return bmdFormat10BitYUV; + default: + assert(false); + } +} + +} // namespace + +DeckLinkCapture::DeckLinkCapture(IDeckLink *card, int card_index) + : card_index(card_index), card(card) +{ + { + const char *model_name; + char buf[256]; + if (card->GetModelName(&model_name) == S_OK) { + snprintf(buf, sizeof(buf), "PCI card %d: %s", card_index, model_name); + } else { + snprintf(buf, sizeof(buf), "PCI card %d: Unknown DeckLink card", card_index); + } + description = buf; + } + + if (card->QueryInterface(IID_IDeckLinkInput, (void**)&input) != S_OK) { + fprintf(stderr, "Card %d has no inputs\n", card_index); + exit(1); + } + + IDeckLinkAttributes *attr; + if (card->QueryInterface(IID_IDeckLinkAttributes, (void**)&attr) != S_OK) { + fprintf(stderr, "Card %d has no attributes\n", card_index); + exit(1); + } + + // Get the list of available video inputs. + int64_t video_input_mask; + if (attr->GetInt(BMDDeckLinkVideoInputConnections, &video_input_mask) != S_OK) { + fprintf(stderr, "Failed to enumerate video inputs for card %d\n", card_index); + exit(1); + } + const vector> video_input_types = { + { bmdVideoConnectionSDI, "SDI" }, + { bmdVideoConnectionHDMI, "HDMI" }, + { bmdVideoConnectionOpticalSDI, "Optical SDI" }, + { bmdVideoConnectionComponent, "Component" }, + { bmdVideoConnectionComposite, "Composite" }, + { bmdVideoConnectionSVideo, "S-Video" } + }; + for (const auto &video_input : video_input_types) { + if (video_input_mask & video_input.first) { + video_inputs.emplace(video_input.first, video_input.second); + } + } + + // And then the available audio inputs. + int64_t audio_input_mask; + if (attr->GetInt(BMDDeckLinkAudioInputConnections, &audio_input_mask) != S_OK) { + fprintf(stderr, "Failed to enumerate audio inputs for card %d\n", card_index); + exit(1); + } + const vector> audio_input_types = { + { bmdAudioConnectionEmbedded, "Embedded" }, + { bmdAudioConnectionAESEBU, "AES/EBU" }, + { bmdAudioConnectionAnalog, "Analog" }, + { bmdAudioConnectionAnalogXLR, "Analog XLR" }, + { bmdAudioConnectionAnalogRCA, "Analog RCA" }, + { bmdAudioConnectionMicrophone, "Microphone" }, + { bmdAudioConnectionHeadphones, "Headphones" } + }; + for (const auto &audio_input : audio_input_types) { + if (audio_input_mask & audio_input.first) { + audio_inputs.emplace(audio_input.first, audio_input.second); + } + } + + // Check if we the card supports input autodetection. + if (attr->GetFlag(BMDDeckLinkSupportsInputFormatDetection, &supports_autodetect) != S_OK) { + fprintf(stderr, "Warning: Failed to ask card %d whether it supports input format autodetection\n", card_index); + supports_autodetect = false; + } + + // If there's more than one subdevice on this card, label them. + int64_t num_subdevices, subdevice_idx; + if (attr->GetInt(BMDDeckLinkNumberOfSubDevices, &num_subdevices) == S_OK && num_subdevices > 1) { + if (attr->GetInt(BMDDeckLinkSubDeviceIndex, &subdevice_idx) == S_OK) { + char buf[256]; + snprintf(buf, sizeof(buf), " (subdevice %d)", int(subdevice_idx)); + description += buf; + } + } + + attr->Release(); + + /* Set up the video and audio sources. */ + if (card->QueryInterface(IID_IDeckLinkConfiguration, (void**)&config) != S_OK) { + fprintf(stderr, "Failed to get configuration interface for card %d\n", card_index); + exit(1); + } + + BMDVideoConnection connection = pick_default_video_connection(card, BMDDeckLinkVideoInputConnections, card_index); + + set_video_input(connection); + set_audio_input(bmdAudioConnectionEmbedded); + + IDeckLinkDisplayModeIterator *mode_it; + if (input->GetDisplayModeIterator(&mode_it) != S_OK) { + fprintf(stderr, "Failed to enumerate display modes for card %d\n", card_index); + exit(1); + } + + video_modes = summarize_video_modes(mode_it, card_index); + mode_it->Release(); + + set_video_mode_no_restart(bmdModeHD720p5994); + + input->SetCallback(this); +} + +DeckLinkCapture::~DeckLinkCapture() +{ + if (has_dequeue_callbacks) { + dequeue_cleanup_callback(); + } + input->Release(); + config->Release(); + card->Release(); +} + +HRESULT STDMETHODCALLTYPE DeckLinkCapture::QueryInterface(REFIID, LPVOID *) +{ + return E_NOINTERFACE; +} + +ULONG STDMETHODCALLTYPE DeckLinkCapture::AddRef(void) +{ + return refcount.fetch_add(1) + 1; +} + +ULONG STDMETHODCALLTYPE DeckLinkCapture::Release(void) +{ + int new_ref = refcount.fetch_sub(1) - 1; + if (new_ref == 0) + delete this; + return new_ref; +} + +HRESULT STDMETHODCALLTYPE DeckLinkCapture::VideoInputFormatChanged( + BMDVideoInputFormatChangedEvents, + IDeckLinkDisplayMode* display_mode, + BMDDetectedVideoInputFormatFlags format_flags) +{ + if (format_flags & bmdDetectedVideoInputRGB444) { + fprintf(stderr, "WARNING: Input detected as 4:4:4 RGB, but Nageru can't consume that yet.\n"); + fprintf(stderr, "Doing hardware conversion to 4:2:2 Y'CbCr.\n"); + } + if (supports_autodetect && display_mode->GetDisplayMode() != current_video_mode) { + set_video_mode(display_mode->GetDisplayMode()); + } + if (display_mode->GetFrameRate(&frame_duration, &time_scale) != S_OK) { + fprintf(stderr, "Could not get new frame rate\n"); + exit(1); + } + field_dominance = display_mode->GetFieldDominance(); + return S_OK; +} + +HRESULT STDMETHODCALLTYPE DeckLinkCapture::VideoInputFrameArrived( + IDeckLinkVideoInputFrame *video_frame, + IDeckLinkAudioInputPacket *audio_frame) +{ + if (!done_init) { + char thread_name[16]; + snprintf(thread_name, sizeof(thread_name), "DeckLink_C_%d", card_index); + pthread_setname_np(pthread_self(), thread_name); + + sched_param param; + memset(¶m, 0, sizeof(param)); + param.sched_priority = 1; + if (sched_setscheduler(0, SCHED_RR, ¶m) == -1) { + printf("couldn't set realtime priority for DeckLink thread: %s\n", strerror(errno)); + } + + if (has_dequeue_callbacks) { + dequeue_init_callback(); + } + done_init = true; + } + + steady_clock::time_point now = steady_clock::now(); + + FrameAllocator::Frame current_video_frame, current_audio_frame; + VideoFormat video_format; + AudioFormat audio_format; + + video_format.frame_rate_nom = time_scale; + video_format.frame_rate_den = frame_duration; + // TODO: Respect the TFF/BFF flag. + video_format.interlaced = (field_dominance == bmdLowerFieldFirst || field_dominance == bmdUpperFieldFirst); + video_format.second_field_start = 1; + + if (video_frame) { + video_format.has_signal = !(video_frame->GetFlags() & bmdFrameHasNoInputSource); + + const int width = video_frame->GetWidth(); + const int height = video_frame->GetHeight(); + const int stride = video_frame->GetRowBytes(); + const BMDPixelFormat format = video_frame->GetPixelFormat(); + assert(format == pixel_format_to_bmd(current_pixel_format)); + if (global_flags.ten_bit_input) { + assert(stride == int(v210Converter::get_v210_stride(width))); + } else { + assert(stride == width * 2); + } + + current_video_frame = video_frame_allocator->alloc_frame(); + if (current_video_frame.data != nullptr) { + const uint8_t *frame_bytes; + video_frame->GetBytes((void **)&frame_bytes); + size_t num_bytes = stride * height; + + if (current_video_frame.interleaved) { + uint8_t *data = current_video_frame.data; + uint8_t *data2 = current_video_frame.data2; + memcpy_interleaved(data, data2, frame_bytes, num_bytes); + } else { + memcpy(current_video_frame.data, frame_bytes, num_bytes); + } + current_video_frame.len += num_bytes; + + video_format.width = width; + video_format.height = height; + video_format.stride = stride; + } + } + + if (audio_frame) { + int num_samples = audio_frame->GetSampleFrameCount(); + + current_audio_frame = audio_frame_allocator->alloc_frame(); + if (current_audio_frame.data != nullptr) { + const uint8_t *frame_bytes; + audio_frame->GetBytes((void **)&frame_bytes); + current_audio_frame.len = sizeof(int32_t) * 2 * num_samples; + + memcpy(current_audio_frame.data, frame_bytes, current_audio_frame.len); + + audio_format.bits_per_sample = 32; + audio_format.num_channels = 2; + } + } + + current_video_frame.received_timestamp = now; + current_audio_frame.received_timestamp = now; + + if (current_video_frame.data != nullptr || current_audio_frame.data != nullptr) { + // TODO: Put into a queue and put into a dequeue thread, if the + // BlackMagic drivers don't already do that for us? + frame_callback(timecode, + current_video_frame, /*video_offset=*/0, video_format, + current_audio_frame, /*audio_offset=*/0, audio_format); + } + + timecode++; + return S_OK; +} + +void DeckLinkCapture::configure_card() +{ + if (video_frame_allocator == nullptr) { + owned_video_frame_allocator.reset(new MallocFrameAllocator(FRAME_SIZE, NUM_QUEUED_VIDEO_FRAMES)); + set_video_frame_allocator(owned_video_frame_allocator.get()); + } + if (audio_frame_allocator == nullptr) { + owned_audio_frame_allocator.reset(new MallocFrameAllocator(65536, NUM_QUEUED_AUDIO_FRAMES)); + set_audio_frame_allocator(owned_audio_frame_allocator.get()); + } +} + +void DeckLinkCapture::start_bm_capture() +{ + if (running) { + return; + } + if (input->EnableVideoInput(current_video_mode, pixel_format_to_bmd(current_pixel_format), supports_autodetect ? bmdVideoInputEnableFormatDetection : 0) != S_OK) { + fprintf(stderr, "Failed to set video mode 0x%04x for card %d\n", current_video_mode, card_index); + exit(1); + } + if (input->EnableAudioInput(48000, bmdAudioSampleType32bitInteger, 2) != S_OK) { + fprintf(stderr, "Failed to enable audio input for card %d\n", card_index); + exit(1); + } + + if (input->StartStreams() != S_OK) { + fprintf(stderr, "StartStreams failed\n"); + exit(1); + } + running = true; +} + +void DeckLinkCapture::stop_dequeue_thread() +{ + if (!running) { + return; + } + HRESULT result = input->StopStreams(); + if (result != S_OK) { + fprintf(stderr, "StopStreams failed with error 0x%x\n", result); + exit(1); + } + + // We could call DisableVideoInput() and DisableAudioInput() here, + // but they seem to be taking a really long time, and we only do this + // during shutdown anyway, so StopStreams() will suffice. + + running = false; +} + +void DeckLinkCapture::set_video_mode(uint32_t video_mode_id) +{ + if (running) { + if (input->PauseStreams() != S_OK) { + fprintf(stderr, "PauseStreams failed\n"); + exit(1); + } + if (input->FlushStreams() != S_OK) { + fprintf(stderr, "FlushStreams failed\n"); + exit(1); + } + } + + set_video_mode_no_restart(video_mode_id); + + if (running) { + if (input->StartStreams() != S_OK) { + fprintf(stderr, "StartStreams failed\n"); + exit(1); + } + } +} + +void DeckLinkCapture::set_pixel_format(PixelFormat pixel_format) +{ + current_pixel_format = pixel_format; + set_video_mode(current_video_mode); +} + +void DeckLinkCapture::set_video_mode_no_restart(uint32_t video_mode_id) +{ + BMDDisplayModeSupport support; + IDeckLinkDisplayMode *display_mode; + if (input->DoesSupportVideoMode(video_mode_id, pixel_format_to_bmd(current_pixel_format), /*flags=*/0, &support, &display_mode)) { + fprintf(stderr, "Failed to query display mode for card %d\n", card_index); + exit(1); + } + + if (support == bmdDisplayModeNotSupported) { + fprintf(stderr, "Card %d does not support display mode\n", card_index); + exit(1); + } + + if (display_mode->GetFrameRate(&frame_duration, &time_scale) != S_OK) { + fprintf(stderr, "Could not get frame rate for card %d\n", card_index); + exit(1); + } + + field_dominance = display_mode->GetFieldDominance(); + + if (running) { + if (input->EnableVideoInput(video_mode_id, pixel_format_to_bmd(current_pixel_format), supports_autodetect ? bmdVideoInputEnableFormatDetection : 0) != S_OK) { + fprintf(stderr, "Failed to set video mode 0x%04x for card %d\n", video_mode_id, card_index); + exit(1); + } + } + + current_video_mode = video_mode_id; +} + +void DeckLinkCapture::set_video_input(uint32_t video_input_id) +{ + if (config->SetInt(bmdDeckLinkConfigVideoInputConnection, video_input_id) != S_OK) { + fprintf(stderr, "Failed to set video input connection for card %d\n", card_index); + exit(1); + } + + current_video_input = video_input_id; +} + +void DeckLinkCapture::set_audio_input(uint32_t audio_input_id) +{ + if (config->SetInt(bmdDeckLinkConfigAudioInputConnection, audio_input_id) != S_OK) { + fprintf(stderr, "Failed to set audio input connection for card %d\n", card_index); + exit(1); + } + + current_audio_input = audio_input_id; +} diff --git a/nageru/decklink_capture.h b/nageru/decklink_capture.h new file mode 100644 index 0000000..f940241 --- /dev/null +++ b/nageru/decklink_capture.h @@ -0,0 +1,153 @@ +#ifndef _DECKLINK_CAPTURE_H +#define _DECKLINK_CAPTURE_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "DeckLinkAPIModes.h" +#include "DeckLinkAPITypes.h" +#include "LinuxCOM.h" +#include "bmusb/bmusb.h" + +class IDeckLink; +class IDeckLinkConfiguration; + +// TODO: Adjust CaptureInterface to be a little less bmusb-centric. +// There are too many member functions here that don't really do anything. +class DeckLinkCapture : public bmusb::CaptureInterface, IDeckLinkInputCallback +{ +public: + DeckLinkCapture(IDeckLink *card, int card_index); // Takes ownership of . + ~DeckLinkCapture(); + + // IDeckLinkInputCallback. + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID *) override; + ULONG STDMETHODCALLTYPE AddRef() override; + ULONG STDMETHODCALLTYPE Release() override; + HRESULT STDMETHODCALLTYPE VideoInputFormatChanged( + BMDVideoInputFormatChangedEvents, + IDeckLinkDisplayMode*, + BMDDetectedVideoInputFormatFlags) override; + HRESULT STDMETHODCALLTYPE VideoInputFrameArrived( + IDeckLinkVideoInputFrame *video_frame, + IDeckLinkAudioInputPacket *audio_frame) override; + + // CaptureInterface. + void set_video_frame_allocator(bmusb::FrameAllocator *allocator) override + { + video_frame_allocator = allocator; + if (owned_video_frame_allocator.get() != allocator) { + owned_video_frame_allocator.reset(); + } + } + + bmusb::FrameAllocator *get_video_frame_allocator() override + { + return video_frame_allocator; + } + + // Does not take ownership. + void set_audio_frame_allocator(bmusb::FrameAllocator *allocator) override + { + audio_frame_allocator = allocator; + if (owned_audio_frame_allocator.get() != allocator) { + owned_audio_frame_allocator.reset(); + } + } + + bmusb::FrameAllocator *get_audio_frame_allocator() override + { + return audio_frame_allocator; + } + + void set_frame_callback(bmusb::frame_callback_t callback) override + { + frame_callback = callback; + } + + void set_dequeue_thread_callbacks(std::function init, std::function cleanup) override + { + dequeue_init_callback = init; + dequeue_cleanup_callback = cleanup; + has_dequeue_callbacks = true; + } + + std::string get_description() const override + { + return description; + } + + void configure_card() override; + void start_bm_capture() override; + void stop_dequeue_thread() override; + + // TODO: Can the API communicate this to us somehow, for e.g. Thunderbolt cards? + bool get_disconnected() const override { return false; } + + std::map get_available_video_modes() const override { return video_modes; } + void set_video_mode(uint32_t video_mode_id) override; + uint32_t get_current_video_mode() const override { return current_video_mode; } + + std::set get_available_pixel_formats() const override { + return std::set{ bmusb::PixelFormat_8BitYCbCr, bmusb::PixelFormat_10BitYCbCr }; + } + void set_pixel_format(bmusb::PixelFormat pixel_format) override; + bmusb::PixelFormat get_current_pixel_format() const override { + return current_pixel_format; + } + + std::map get_available_video_inputs() const override { return video_inputs; } + void set_video_input(uint32_t video_input_id) override; + uint32_t get_current_video_input() const override { return current_video_input; } + + std::map get_available_audio_inputs() const override { return audio_inputs; } + void set_audio_input(uint32_t audio_input_id) override; + uint32_t get_current_audio_input() const override { return current_audio_input; } + +private: + void set_video_mode_no_restart(uint32_t video_mode_id); + + std::atomic refcount{1}; + bool done_init = false; + std::string description; + uint16_t timecode = 0; + int card_index; + + bool has_dequeue_callbacks = false; + std::function dequeue_init_callback = nullptr; + std::function dequeue_cleanup_callback = nullptr; + + bmusb::FrameAllocator *video_frame_allocator = nullptr; + bmusb::FrameAllocator *audio_frame_allocator = nullptr; + std::unique_ptr owned_video_frame_allocator; + std::unique_ptr owned_audio_frame_allocator; + bmusb::frame_callback_t frame_callback = nullptr; + + IDeckLinkConfiguration *config = nullptr; + + IDeckLink *card = nullptr; + IDeckLinkInput *input = nullptr; + BMDTimeValue frame_duration; + BMDTimeScale time_scale; + BMDFieldDominance field_dominance; + bool running = false; + bool supports_autodetect = false; + + std::map video_modes; + BMDDisplayMode current_video_mode; + bmusb::PixelFormat current_pixel_format = bmusb::PixelFormat_8BitYCbCr; + + std::map video_inputs; + BMDVideoConnection current_video_input; + + std::map audio_inputs; + BMDAudioConnection current_audio_input; +}; + +#endif // !defined(_DECKLINK_CAPTURE_H) diff --git a/nageru/decklink_output.cpp b/nageru/decklink_output.cpp new file mode 100644 index 0000000..28f433a --- /dev/null +++ b/nageru/decklink_output.cpp @@ -0,0 +1,695 @@ +#include +#include +#include // Must be above the Xlib includes. +#include +#include + +#include + +#include + +#include "chroma_subsampler.h" +#include "decklink_output.h" +#include "decklink_util.h" +#include "flags.h" +#include "metrics.h" +#include "print_latency.h" +#include "timebase.h" +#include "v210_converter.h" + +using namespace movit; +using namespace std; +using namespace std::chrono; + +namespace { + +// This class can be deleted during regular use, so make all the metrics static. +once_flag decklink_metrics_inited; +LatencyHistogram latency_histogram; +atomic metric_decklink_output_width_pixels{-1}; +atomic metric_decklink_output_height_pixels{-1}; +atomic metric_decklink_output_frame_rate_den{-1}; +atomic metric_decklink_output_frame_rate_nom{-1}; +atomic metric_decklink_output_inflight_frames{0}; +atomic metric_decklink_output_color_mismatch_frames{0}; + +atomic metric_decklink_output_scheduled_frames_dropped{0}; +atomic metric_decklink_output_scheduled_frames_late{0}; +atomic metric_decklink_output_scheduled_frames_normal{0}; +atomic metric_decklink_output_scheduled_frames_preroll{0}; + +atomic metric_decklink_output_completed_frames_completed{0}; +atomic metric_decklink_output_completed_frames_dropped{0}; +atomic metric_decklink_output_completed_frames_flushed{0}; +atomic metric_decklink_output_completed_frames_late{0}; +atomic metric_decklink_output_completed_frames_unknown{0}; + +atomic metric_decklink_output_scheduled_samples{0}; + +Summary metric_decklink_output_margin_seconds; + +} // namespace + +DeckLinkOutput::DeckLinkOutput(ResourcePool *resource_pool, QSurface *surface, unsigned width, unsigned height, unsigned card_index) + : resource_pool(resource_pool), surface(surface), width(width), height(height), card_index(card_index) +{ + chroma_subsampler.reset(new ChromaSubsampler(resource_pool)); + + call_once(decklink_metrics_inited, [](){ + latency_histogram.init("decklink_output"); + global_metrics.add("decklink_output_width_pixels", &metric_decklink_output_width_pixels, Metrics::TYPE_GAUGE); + global_metrics.add("decklink_output_height_pixels", &metric_decklink_output_height_pixels, Metrics::TYPE_GAUGE); + global_metrics.add("decklink_output_frame_rate_den", &metric_decklink_output_frame_rate_den, Metrics::TYPE_GAUGE); + global_metrics.add("decklink_output_frame_rate_nom", &metric_decklink_output_frame_rate_nom, Metrics::TYPE_GAUGE); + global_metrics.add("decklink_output_inflight_frames", &metric_decklink_output_inflight_frames, Metrics::TYPE_GAUGE); + global_metrics.add("decklink_output_color_mismatch_frames", &metric_decklink_output_color_mismatch_frames); + + global_metrics.add("decklink_output_scheduled_frames", {{ "status", "dropped" }}, &metric_decklink_output_scheduled_frames_dropped); + global_metrics.add("decklink_output_scheduled_frames", {{ "status", "late" }}, &metric_decklink_output_scheduled_frames_late); + global_metrics.add("decklink_output_scheduled_frames", {{ "status", "normal" }}, &metric_decklink_output_scheduled_frames_normal); + global_metrics.add("decklink_output_scheduled_frames", {{ "status", "preroll" }}, &metric_decklink_output_scheduled_frames_preroll); + + global_metrics.add("decklink_output_completed_frames", {{ "status", "completed" }}, &metric_decklink_output_completed_frames_completed); + global_metrics.add("decklink_output_completed_frames", {{ "status", "dropped" }}, &metric_decklink_output_completed_frames_dropped); + global_metrics.add("decklink_output_completed_frames", {{ "status", "flushed" }}, &metric_decklink_output_completed_frames_flushed); + global_metrics.add("decklink_output_completed_frames", {{ "status", "late" }}, &metric_decklink_output_completed_frames_late); + global_metrics.add("decklink_output_completed_frames", {{ "status", "unknown" }}, &metric_decklink_output_completed_frames_unknown); + + global_metrics.add("decklink_output_scheduled_samples", &metric_decklink_output_scheduled_samples); + vector quantiles{0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99}; + metric_decklink_output_margin_seconds.init(quantiles, 60.0); + global_metrics.add("decklink_output_margin_seconds", &metric_decklink_output_margin_seconds); + }); +} + +bool DeckLinkOutput::set_device(IDeckLink *decklink) +{ + if (decklink->QueryInterface(IID_IDeckLinkOutput, (void**)&output) != S_OK) { + fprintf(stderr, "Warning: Card %u has no outputs\n", card_index); + return false; + } + + IDeckLinkDisplayModeIterator *mode_it; + if (output->GetDisplayModeIterator(&mode_it) != S_OK) { + fprintf(stderr, "Warning: Failed to enumerate output display modes for card %u\n", card_index); + return false; + } + + video_modes.clear(); + + for (const auto &it : summarize_video_modes(mode_it, card_index)) { + if (it.second.width != width || it.second.height != height) { + continue; + } + + // We could support interlaced modes, but let's stay out of it for now, + // since we don't have interlaced stream output. + if (it.second.interlaced) { + continue; + } + + video_modes.insert(it); + } + + mode_it->Release(); + + // HDMI or SDI generally mean “both HDMI and SDI at the same time” on DeckLink cards + // that support both; pick_default_video_connection() will generally pick one of those + // if they exist. (--prefer-hdmi-input would also affect the selection despite the name + // of the option, but since either generally means both, it's inconsequential.) + // We're not very likely to need analog outputs, so we don't need a way to change + // beyond that. + video_connection = pick_default_video_connection(decklink, BMDDeckLinkVideoOutputConnections, card_index); + return true; +} + +void DeckLinkOutput::start_output(uint32_t mode, int64_t base_pts) +{ + assert(output); + assert(!playback_initiated); + + if (video_modes.empty()) { + fprintf(stderr, "ERROR: No matching output modes for %dx%d found\n", width, height); + exit(1); + } + + should_quit.unquit(); + playback_initiated = true; + playback_started = false; + this->base_pts = base_pts; + + IDeckLinkConfiguration *config = nullptr; + if (output->QueryInterface(IID_IDeckLinkConfiguration, (void**)&config) != S_OK) { + fprintf(stderr, "Failed to get configuration interface for output card\n"); + exit(1); + } + if (config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true) != S_OK) { + fprintf(stderr, "Failed to set low latency output\n"); + exit(1); + } + if (config->SetInt(bmdDeckLinkConfigVideoOutputConnection, video_connection) != S_OK) { + fprintf(stderr, "Failed to set video output connection for card %u\n", card_index); + exit(1); + } + if (config->SetFlag(bmdDeckLinkConfigUse1080pNotPsF, true) != S_OK) { + fprintf(stderr, "Failed to set PsF flag for card\n"); + exit(1); + } + if (config->SetFlag(bmdDeckLinkConfigSMPTELevelAOutput, true) != S_OK) { + // This affects at least some no-name SDI->HDMI converters. + // Warn, but don't die. + fprintf(stderr, "WARNING: Failed to enable SMTPE Level A; resolutions like 1080p60 might have issues.\n"); + } + + BMDDisplayModeSupport support; + IDeckLinkDisplayMode *display_mode; + BMDPixelFormat pixel_format = global_flags.ten_bit_output ? bmdFormat10BitYUV : bmdFormat8BitYUV; + if (output->DoesSupportVideoMode(mode, pixel_format, bmdVideoOutputFlagDefault, + &support, &display_mode) != S_OK) { + fprintf(stderr, "Couldn't ask for format support\n"); + exit(1); + } + + if (support == bmdDisplayModeNotSupported) { + fprintf(stderr, "Requested display mode not supported\n"); + exit(1); + } + + current_mode_flags = display_mode->GetFlags(); + + BMDTimeValue time_value; + BMDTimeScale time_scale; + if (display_mode->GetFrameRate(&time_value, &time_scale) != S_OK) { + fprintf(stderr, "Couldn't get frame rate\n"); + exit(1); + } + + metric_decklink_output_width_pixels = width; + metric_decklink_output_height_pixels = height; + metric_decklink_output_frame_rate_nom = time_value; + metric_decklink_output_frame_rate_den = time_scale; + + frame_duration = time_value * TIMEBASE / time_scale; + + display_mode->Release(); + + HRESULT result = output->EnableVideoOutput(mode, bmdVideoOutputFlagDefault); + if (result != S_OK) { + fprintf(stderr, "Couldn't enable output with error 0x%x\n", result); + exit(1); + } + if (output->SetScheduledFrameCompletionCallback(this) != S_OK) { + fprintf(stderr, "Couldn't set callback\n"); + exit(1); + } + assert(OUTPUT_FREQUENCY == 48000); + if (output->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped) != S_OK) { + fprintf(stderr, "Couldn't enable audio output\n"); + exit(1); + } + if (output->BeginAudioPreroll() != S_OK) { + fprintf(stderr, "Couldn't begin audio preroll\n"); + exit(1); + } + + present_thread = thread([this]{ + QOpenGLContext *context = create_context(this->surface); + eglBindAPI(EGL_OPENGL_API); + if (!make_current(context, this->surface)) { + printf("display=%p surface=%p context=%p curr=%p err=%d\n", eglGetCurrentDisplay(), this->surface, context, eglGetCurrentContext(), + eglGetError()); + exit(1); + } + present_thread_func(); + delete_context(context); + }); +} + +void DeckLinkOutput::end_output() +{ + if (!playback_initiated) { + return; + } + + should_quit.quit(); + frame_queues_changed.notify_all(); + present_thread.join(); + playback_initiated = false; + + output->StopScheduledPlayback(0, nullptr, 0); + output->DisableVideoOutput(); + output->DisableAudioOutput(); + + // Wait until all frames are accounted for, and free them. + { + unique_lock lock(frame_queue_mutex); + while (!(frame_freelist.empty() && num_frames_in_flight == 0)) { + frame_queues_changed.wait(lock, [this]{ return !frame_freelist.empty(); }); + frame_freelist.pop(); + } + } +} + +void DeckLinkOutput::send_frame(GLuint y_tex, GLuint cbcr_tex, YCbCrLumaCoefficients output_ycbcr_coefficients, const vector &input_frames, int64_t pts, int64_t duration) +{ + assert(!should_quit.should_quit()); + + if ((current_mode_flags & bmdDisplayModeColorspaceRec601) && output_ycbcr_coefficients == YCBCR_REC_709) { + if (!last_frame_had_mode_mismatch) { + fprintf(stderr, "WARNING: Chosen output mode expects Rec. 601 Y'CbCr coefficients.\n"); + fprintf(stderr, " Consider --output-ycbcr-coefficients=rec601 (or =auto).\n"); + } + last_frame_had_mode_mismatch = true; + ++metric_decklink_output_color_mismatch_frames; + } else if ((current_mode_flags & bmdDisplayModeColorspaceRec709) && output_ycbcr_coefficients == YCBCR_REC_601) { + if (!last_frame_had_mode_mismatch) { + fprintf(stderr, "WARNING: Chosen output mode expects Rec. 709 Y'CbCr coefficients.\n"); + fprintf(stderr, " Consider --output-ycbcr-coefficients=rec709 (or =auto).\n"); + } + last_frame_had_mode_mismatch = true; + ++metric_decklink_output_color_mismatch_frames; + } else { + last_frame_had_mode_mismatch = false; + } + + unique_ptr frame = get_frame(); + if (global_flags.ten_bit_output) { + chroma_subsampler->create_v210(y_tex, cbcr_tex, width, height, frame->uyvy_tex); + } else { + chroma_subsampler->create_uyvy(y_tex, cbcr_tex, width, height, frame->uyvy_tex); + } + + // Download the UYVY texture to the PBO. + glPixelStorei(GL_PACK_ROW_LENGTH, 0); + check_error(); + + glBindBuffer(GL_PIXEL_PACK_BUFFER, frame->pbo); + check_error(); + + if (global_flags.ten_bit_output) { + glBindTexture(GL_TEXTURE_2D, frame->uyvy_tex); + check_error(); + glGetTexImage(GL_TEXTURE_2D, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, BUFFER_OFFSET(0)); + check_error(); + } else { + glBindTexture(GL_TEXTURE_2D, frame->uyvy_tex); + check_error(); + glGetTexImage(GL_TEXTURE_2D, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, BUFFER_OFFSET(0)); + check_error(); + } + + glBindTexture(GL_TEXTURE_2D, 0); + check_error(); + glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); + check_error(); + + glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT | GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT); + check_error(); + + frame->fence = RefCountedGLsync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0); + check_error(); + glFlush(); // Make the DeckLink thread see the fence as soon as possible. + check_error(); + + frame->input_frames = input_frames; + frame->received_ts = find_received_timestamp(input_frames); + frame->pts = pts; + frame->duration = duration; + + { + unique_lock lock(frame_queue_mutex); + pending_video_frames.push(move(frame)); + } + frame_queues_changed.notify_all(); +} + +void DeckLinkOutput::send_audio(int64_t pts, const std::vector &samples) +{ + unique_ptr int_samples(new int32_t[samples.size()]); + for (size_t i = 0; i < samples.size(); ++i) { + int_samples[i] = lrintf(samples[i] * 2147483648.0f); + } + + uint32_t frames_written; + HRESULT result = output->ScheduleAudioSamples(int_samples.get(), samples.size() / 2, + pts, TIMEBASE, &frames_written); + if (result != S_OK) { + fprintf(stderr, "ScheduleAudioSamples(pts=%ld) failed (result=0x%08x)\n", pts, result); + } else { + if (frames_written != samples.size() / 2) { + fprintf(stderr, "ScheduleAudioSamples() returned short write (%u/%ld)\n", frames_written, samples.size() / 2); + } + } + metric_decklink_output_scheduled_samples += samples.size() / 2; +} + +void DeckLinkOutput::wait_for_frame(int64_t pts, int *dropped_frames, int64_t *frame_duration, bool *is_preroll, steady_clock::time_point *frame_timestamp) +{ + assert(!should_quit.should_quit()); + + *dropped_frames = 0; + *frame_duration = this->frame_duration; + + const BMDTimeValue buffer = lrint(*frame_duration * global_flags.output_buffer_frames); + const BMDTimeValue max_overshoot = lrint(*frame_duration * global_flags.output_slop_frames); + BMDTimeValue target_time = pts - buffer; + + // While prerolling, we send out frames as quickly as we can. + if (target_time < base_pts) { + *is_preroll = true; + ++metric_decklink_output_scheduled_frames_preroll; + return; + } + + *is_preroll = !playback_started; + + if (!playback_started) { + if (output->EndAudioPreroll() != S_OK) { + fprintf(stderr, "Could not end audio preroll\n"); + exit(1); // TODO + } + if (output->StartScheduledPlayback(base_pts, TIMEBASE, 1.0) != S_OK) { + fprintf(stderr, "Could not start playback\n"); + exit(1); // TODO + } + playback_started = true; + } + + BMDTimeValue stream_frame_time; + double playback_speed; + output->GetScheduledStreamTime(TIMEBASE, &stream_frame_time, &playback_speed); + + *frame_timestamp = steady_clock::now() + + nanoseconds((target_time - stream_frame_time) * 1000000000 / TIMEBASE); + + metric_decklink_output_margin_seconds.count_event( + (target_time - stream_frame_time) / double(TIMEBASE)); + + // If we're ahead of time, wait for the frame to (approximately) start. + if (stream_frame_time < target_time) { + should_quit.sleep_until(*frame_timestamp); + ++metric_decklink_output_scheduled_frames_normal; + return; + } + + // If we overshot the previous frame by just a little, + // fire off one immediately. + if (stream_frame_time < target_time + max_overshoot) { + fprintf(stderr, "Warning: Frame was %ld ms late (but not skipping it due to --output-slop-frames).\n", + lrint((stream_frame_time - target_time) * 1000.0 / TIMEBASE)); + ++metric_decklink_output_scheduled_frames_late; + return; + } + + // Oops, we missed by more than one frame. Return immediately, + // but drop so that we catch up. + *dropped_frames = (stream_frame_time - target_time + *frame_duration - 1) / *frame_duration; + const int64_t ns_per_frame = this->frame_duration * 1000000000 / TIMEBASE; + *frame_timestamp += nanoseconds(*dropped_frames * ns_per_frame); + fprintf(stderr, "Dropped %d output frames; skipping.\n", *dropped_frames); + metric_decklink_output_scheduled_frames_dropped += *dropped_frames; + ++metric_decklink_output_scheduled_frames_normal; +} + +uint32_t DeckLinkOutput::pick_video_mode(uint32_t mode) const +{ + if (video_modes.count(mode)) { + return mode; + } + + // Prioritize 59.94 > 60 > 29.97. If none of those are found, then pick the highest one. + for (const pair &desired : vector>{ { 60000, 1001 }, { 60, 0 }, { 30000, 1001 } }) { + for (const auto &it : video_modes) { + if (it.second.frame_rate_num * desired.second == desired.first * it.second.frame_rate_den) { + return it.first; + } + } + } + + uint32_t best_mode = 0; + double best_fps = 0.0; + for (const auto &it : video_modes) { + double fps = double(it.second.frame_rate_num) / it.second.frame_rate_den; + if (fps > best_fps) { + best_mode = it.first; + best_fps = fps; + } + } + return best_mode; +} + +YCbCrLumaCoefficients DeckLinkOutput::preferred_ycbcr_coefficients() const +{ + if (current_mode_flags & bmdDisplayModeColorspaceRec601) { + return YCBCR_REC_601; + } else { + // Don't bother checking bmdDisplayModeColorspaceRec709; + // if none is set, 709 is a good default anyway. + return YCBCR_REC_709; + } +} + +HRESULT DeckLinkOutput::ScheduledFrameCompleted(/* in */ IDeckLinkVideoFrame *completedFrame, /* in */ BMDOutputFrameCompletionResult result) +{ + Frame *frame = static_cast(completedFrame); + switch (result) { + case bmdOutputFrameCompleted: + ++metric_decklink_output_completed_frames_completed; + break; + case bmdOutputFrameDisplayedLate: + fprintf(stderr, "Output frame displayed late (pts=%ld)\n", frame->pts); + fprintf(stderr, "Consider increasing --output-buffer-frames if this persists.\n"); + ++metric_decklink_output_completed_frames_late; + break; + case bmdOutputFrameDropped: + fprintf(stderr, "Output frame was dropped (pts=%ld)\n", frame->pts); + fprintf(stderr, "Consider increasing --output-buffer-frames if this persists.\n"); + ++metric_decklink_output_completed_frames_dropped; + break; + case bmdOutputFrameFlushed: + fprintf(stderr, "Output frame was flushed (pts=%ld)\n", frame->pts); + ++metric_decklink_output_completed_frames_flushed; + break; + default: + fprintf(stderr, "Output frame completed with unknown status %d\n", result); + ++metric_decklink_output_completed_frames_unknown; + break; + } + + static int frameno = 0; + print_latency("DeckLink output latency (frame received → output on HDMI):", frame->received_ts, false, &frameno, &latency_histogram); + + { + lock_guard lock(frame_queue_mutex); + frame_freelist.push(unique_ptr(frame)); + --num_frames_in_flight; + --metric_decklink_output_inflight_frames; + } + + return S_OK; +} + +HRESULT DeckLinkOutput::ScheduledPlaybackHasStopped() +{ + printf("playback stopped!\n"); + return S_OK; +} + +unique_ptr DeckLinkOutput::get_frame() +{ + lock_guard lock(frame_queue_mutex); + + if (!frame_freelist.empty()) { + unique_ptr frame = move(frame_freelist.front()); + frame_freelist.pop(); + return frame; + } + + unique_ptr frame(new Frame); + + size_t stride; + if (global_flags.ten_bit_output) { + stride = v210Converter::get_v210_stride(width); + GLint v210_width = stride / sizeof(uint32_t); + frame->uyvy_tex = resource_pool->create_2d_texture(GL_RGB10_A2, v210_width, height); + + // We need valid texture state, or NVIDIA won't allow us to write to the texture. + glBindTexture(GL_TEXTURE_2D, frame->uyvy_tex); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + } else { + stride = width * 2; + frame->uyvy_tex = resource_pool->create_2d_texture(GL_RGBA8, width / 2, height); + } + + glGenBuffers(1, &frame->pbo); + check_error(); + glBindBuffer(GL_PIXEL_PACK_BUFFER, frame->pbo); + check_error(); + glBufferStorage(GL_PIXEL_PACK_BUFFER, stride * height, nullptr, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT); + check_error(); + frame->uyvy_ptr = (uint8_t *)glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, stride * height, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT); + check_error(); + frame->uyvy_ptr_local.reset(new uint8_t[stride * height]); + frame->resource_pool = resource_pool; + + return frame; +} + +void DeckLinkOutput::present_thread_func() +{ + pthread_setname_np(pthread_self(), "DeckLinkOutput"); + for ( ;; ) { + unique_ptr frame; + { + unique_lock lock(frame_queue_mutex); + frame_queues_changed.wait(lock, [this]{ + return should_quit.should_quit() || !pending_video_frames.empty(); + }); + if (should_quit.should_quit()) { + return; + } + frame = move(pending_video_frames.front()); + pending_video_frames.pop(); + ++num_frames_in_flight; + ++metric_decklink_output_inflight_frames; + } + + for ( ;; ) { + int err = glClientWaitSync(frame->fence.get(), /*flags=*/0, 0); + if (err == GL_TIMEOUT_EXPIRED) { + // NVIDIA likes to busy-wait; yield instead. + this_thread::sleep_for(milliseconds(1)); + } else { + break; + } + } + check_error(); + frame->fence.reset(); + + if (global_flags.ten_bit_output) { + memcpy(frame->uyvy_ptr_local.get(), frame->uyvy_ptr, v210Converter::get_v210_stride(width) * height); + } else { + memcpy(frame->uyvy_ptr_local.get(), frame->uyvy_ptr, width * height * 2); + } + + // Release any input frames we needed to render this frame. + frame->input_frames.clear(); + + BMDTimeValue pts = frame->pts; + BMDTimeValue duration = frame->duration; + HRESULT res = output->ScheduleVideoFrame(frame.get(), pts, duration, TIMEBASE); + if (res == S_OK) { + frame.release(); // Owned by the driver now. + } else { + fprintf(stderr, "Could not schedule video frame! (error=0x%08x)\n", res); + + lock_guard lock(frame_queue_mutex); + frame_freelist.push(move(frame)); + --num_frames_in_flight; + --metric_decklink_output_inflight_frames; + } + } +} + +HRESULT STDMETHODCALLTYPE DeckLinkOutput::QueryInterface(REFIID, LPVOID *) +{ + return E_NOINTERFACE; +} + +ULONG STDMETHODCALLTYPE DeckLinkOutput::AddRef() +{ + return refcount.fetch_add(1) + 1; +} + +ULONG STDMETHODCALLTYPE DeckLinkOutput::Release() +{ + int new_ref = refcount.fetch_sub(1) - 1; + if (new_ref == 0) + delete this; + return new_ref; +} + +DeckLinkOutput::Frame::~Frame() +{ + glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo); + check_error(); + glUnmapBuffer(GL_PIXEL_PACK_BUFFER); + check_error(); + glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); + check_error(); + glDeleteBuffers(1, &pbo); + check_error(); + resource_pool->release_2d_texture(uyvy_tex); + check_error(); +} + +HRESULT STDMETHODCALLTYPE DeckLinkOutput::Frame::QueryInterface(REFIID, LPVOID *) +{ + return E_NOINTERFACE; +} + +ULONG STDMETHODCALLTYPE DeckLinkOutput::Frame::AddRef() +{ + return refcount.fetch_add(1) + 1; +} + +ULONG STDMETHODCALLTYPE DeckLinkOutput::Frame::Release() +{ + int new_ref = refcount.fetch_sub(1) - 1; + if (new_ref == 0) + delete this; + return new_ref; +} + +long DeckLinkOutput::Frame::GetWidth() +{ + return global_flags.width; +} + +long DeckLinkOutput::Frame::GetHeight() +{ + return global_flags.height; +} + +long DeckLinkOutput::Frame::GetRowBytes() +{ + if (global_flags.ten_bit_output) { + return v210Converter::get_v210_stride(global_flags.width); + } else { + return global_flags.width * 2; + } +} + +BMDPixelFormat DeckLinkOutput::Frame::GetPixelFormat() +{ + if (global_flags.ten_bit_output) { + return bmdFormat10BitYUV; + } else { + return bmdFormat8BitYUV; + } +} + +BMDFrameFlags DeckLinkOutput::Frame::GetFlags() +{ + return bmdFrameFlagDefault; +} + +HRESULT DeckLinkOutput::Frame::GetBytes(/* out */ void **buffer) +{ + *buffer = uyvy_ptr_local.get(); + return S_OK; +} + +HRESULT DeckLinkOutput::Frame::GetTimecode(/* in */ BMDTimecodeFormat format, /* out */ IDeckLinkTimecode **timecode) +{ + fprintf(stderr, "STUB: GetTimecode()\n"); + return E_NOTIMPL; +} + +HRESULT DeckLinkOutput::Frame::GetAncillaryData(/* out */ IDeckLinkVideoFrameAncillary **ancillary) +{ + fprintf(stderr, "STUB: GetAncillaryData()\n"); + return E_NOTIMPL; +} diff --git a/nageru/decklink_output.h b/nageru/decklink_output.h new file mode 100644 index 0000000..44eb86d --- /dev/null +++ b/nageru/decklink_output.h @@ -0,0 +1,155 @@ +#ifndef _DECKLINK_OUTPUT_H +#define _DECKLINK_OUTPUT_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "DeckLinkAPI.h" +#include "DeckLinkAPITypes.h" +#include "LinuxCOM.h" + +#include "context.h" +#include "print_latency.h" +#include "quittable_sleeper.h" +#include "ref_counted_frame.h" +#include "ref_counted_gl_sync.h" + +namespace movit { + +class ResourcePool; + +} // namespace movit + +class ChromaSubsampler; +class IDeckLink; +class IDeckLinkOutput; +class QSurface; + +class DeckLinkOutput : public IDeckLinkVideoOutputCallback { +public: + DeckLinkOutput(movit::ResourcePool *resource_pool, QSurface *surface, unsigned width, unsigned height, unsigned card_index); + + bool set_device(IDeckLink *output); + void start_output(uint32_t mode, int64_t base_pts); // Mode comes from get_available_video_modes(). + void end_output(); + + void send_frame(GLuint y_tex, GLuint cbcr_tex, movit::YCbCrLumaCoefficients ycbcr_coefficients, const std::vector &input_frames, int64_t pts, int64_t duration); + void send_audio(int64_t pts, const std::vector &samples); + + // NOTE: The returned timestamp is undefined for preroll. + // Otherwise, it is the timestamp of the output frame as it should have been, + // even if we're overshooting. E.g. at 50 fps (0.02 spf), assuming the + // last frame was at t=0.980: + // + // If we're at t=0.999, we wait until t=1.000 and return that. + // If we're at t=1.001, we return t=1.000 immediately (small overshoot). + // If we're at t=1.055, we drop two frames and return t=1.040 immediately. + void wait_for_frame(int64_t pts, int *dropped_frames, int64_t *frame_duration, bool *is_preroll, std::chrono::steady_clock::time_point *frame_timestamp); + + // Analogous to CaptureInterface. Will only return modes that have the right width/height. + std::map get_available_video_modes() const { return video_modes; } + + // If the given mode is supported, return it. If not, pick some “best” valid mode. + uint32_t pick_video_mode(uint32_t mode) const; + + // Desired Y'CbCr coefficients for the current mode. Undefined before start_output(). + movit::YCbCrLumaCoefficients preferred_ycbcr_coefficients() const; + + // IUnknown. + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv) override; + ULONG STDMETHODCALLTYPE AddRef() override; + ULONG STDMETHODCALLTYPE Release() override; + + // IDeckLinkVideoOutputCallback. + HRESULT ScheduledFrameCompleted(/* in */ IDeckLinkVideoFrame *completedFrame, /* in */ BMDOutputFrameCompletionResult result) override; + HRESULT ScheduledPlaybackHasStopped() override; + +private: + struct Frame : public IDeckLinkVideoFrame { + public: + ~Frame(); + + // IUnknown. + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv) override; + ULONG STDMETHODCALLTYPE AddRef() override; + ULONG STDMETHODCALLTYPE Release() override; + + // IDeckLinkVideoFrame. + long GetWidth() override; + long GetHeight() override; + long GetRowBytes() override; + BMDPixelFormat GetPixelFormat() override; + BMDFrameFlags GetFlags() override; + HRESULT GetBytes(/* out */ void **buffer) override; + + HRESULT GetTimecode(/* in */ BMDTimecodeFormat format, /* out */ IDeckLinkTimecode **timecode) override; + HRESULT GetAncillaryData(/* out */ IDeckLinkVideoFrameAncillary **ancillary) override; + + private: + std::atomic refcount{1}; + RefCountedGLsync fence; // Needs to be waited on before uyvy_ptr can be read from. + std::vector input_frames; // Cannot be released before we are done rendering (ie., is asserted). + ReceivedTimestamps received_ts; + int64_t pts, duration; + movit::ResourcePool *resource_pool; + + // These members are persistently allocated, and reused when the frame object is. + GLuint uyvy_tex; // Owned by . Can also hold v210 data. + GLuint pbo; + uint8_t *uyvy_ptr; // Persistent mapping into the PBO. + + // Current Blackmagic drivers (January 2017) have a bug where sending a PBO + // pointer to the driver causes a kernel oops. Thus, we do an extra copy into + // this pointer before giving the data to the driver. (We don't do a get + // directly into this pointer, because e.g. Intel/Mesa hits a slow path when + // you do readback into something that's not a PBO.) When Blackmagic fixes + // the bug, we should drop this. + std::unique_ptr uyvy_ptr_local; + + friend class DeckLinkOutput; + }; + std::unique_ptr get_frame(); + void create_uyvy(GLuint y_tex, GLuint cbcr_tex, GLuint dst_tex); + + void present_thread_func(); + + std::atomic refcount{1}; + + std::unique_ptr chroma_subsampler; + std::map video_modes; + + std::thread present_thread; + QuittableSleeper should_quit; + + std::mutex frame_queue_mutex; + std::queue> pending_video_frames; // Under . + std::queue> frame_freelist; // Under . + int num_frames_in_flight = 0; // Number of frames allocated but not on the freelist. Under . + std::condition_variable frame_queues_changed; + bool playback_initiated = false, playback_started = false; + int64_t base_pts, frame_duration; + BMDDisplayModeFlags current_mode_flags = 0; + bool last_frame_had_mode_mismatch = false; + + movit::ResourcePool *resource_pool; + IDeckLinkOutput *output = nullptr; + BMDVideoConnection video_connection; + QSurface *surface; + unsigned width, height; + unsigned card_index; + + GLuint uyvy_vbo; // Holds position and texcoord data. + GLuint uyvy_program_num; // Owned by . + GLuint uyvy_position_attribute_index, uyvy_texcoord_attribute_index; +}; + +#endif // !defined(_DECKLINK_OUTPUT_H) diff --git a/nageru/decklink_util.cpp b/nageru/decklink_util.cpp new file mode 100644 index 0000000..4b701ab --- /dev/null +++ b/nageru/decklink_util.cpp @@ -0,0 +1,92 @@ +#include +#include + +#include + +#include "decklink_util.h" +#include "flags.h" + +using namespace bmusb; +using namespace std; + +map summarize_video_modes(IDeckLinkDisplayModeIterator *mode_it, unsigned card_index) +{ + map video_modes; + + for (IDeckLinkDisplayMode *mode_ptr; mode_it->Next(&mode_ptr) == S_OK; mode_ptr->Release()) { + VideoMode mode; + + const char *mode_name; + if (mode_ptr->GetName(&mode_name) != S_OK) { + mode.name = "Unknown mode"; + } else { + mode.name = mode_name; + free((char *)mode_name); + } + + mode.autodetect = false; + + mode.width = mode_ptr->GetWidth(); + mode.height = mode_ptr->GetHeight(); + + BMDTimeScale frame_rate_num; + BMDTimeValue frame_rate_den; + if (mode_ptr->GetFrameRate(&frame_rate_den, &frame_rate_num) != S_OK) { + fprintf(stderr, "Could not get frame rate for mode '%s' on card %d\n", mode.name.c_str(), card_index); + exit(1); + } + mode.frame_rate_num = frame_rate_num; + mode.frame_rate_den = frame_rate_den; + + // TODO: Respect the TFF/BFF flag. + mode.interlaced = (mode_ptr->GetFieldDominance() == bmdLowerFieldFirst || mode_ptr->GetFieldDominance() == bmdUpperFieldFirst); + + uint32_t id = mode_ptr->GetDisplayMode(); + video_modes.insert(make_pair(id, mode)); + } + + return video_modes; +} + +BMDVideoConnection pick_default_video_connection(IDeckLink *card, BMDDeckLinkAttributeID attribute_id, unsigned card_index) +{ + assert(attribute_id == BMDDeckLinkVideoInputConnections || + attribute_id == BMDDeckLinkVideoOutputConnections); + + IDeckLinkAttributes *attr; + if (card->QueryInterface(IID_IDeckLinkAttributes, (void**)&attr) != S_OK) { + fprintf(stderr, "Card %u has no attributes\n", card_index); + exit(1); + } + + int64_t connection_mask; + if (attr->GetInt(attribute_id, &connection_mask) != S_OK) { + if (attribute_id == BMDDeckLinkVideoInputConnections) { + fprintf(stderr, "Failed to enumerate video inputs for card %u\n", card_index); + } else { + fprintf(stderr, "Failed to enumerate video outputs for card %u\n", card_index); + } + exit(1); + } + attr->Release(); + if (connection_mask == 0) { + if (attribute_id == BMDDeckLinkVideoInputConnections) { + fprintf(stderr, "Card %u has no input connections\n", card_index); + } else { + fprintf(stderr, "Card %u has no output connections\n", card_index); + } + exit(1); + } + + if ((connection_mask & bmdVideoConnectionHDMI) && + global_flags.default_hdmi_input) { + return bmdVideoConnectionHDMI; + } else if (connection_mask & bmdVideoConnectionSDI) { + return bmdVideoConnectionSDI; + } else if (connection_mask & bmdVideoConnectionHDMI) { + return bmdVideoConnectionHDMI; + } else { + // Fallback: Return lowest-set bit, whatever that might be. + return connection_mask & (-connection_mask); + } +} diff --git a/nageru/decklink_util.h b/nageru/decklink_util.h new file mode 100644 index 0000000..2850a21 --- /dev/null +++ b/nageru/decklink_util.h @@ -0,0 +1,17 @@ +#ifndef _DECKLINK_UTIL +#define _DECKLINK_UTIL 1 + +#include + +#include + +#include "bmusb/bmusb.h" + +class IDeckLinkDisplayModeIterator; + +std::map summarize_video_modes(IDeckLinkDisplayModeIterator *mode_it, unsigned card_index); + +// Picks a video connection that the card supports. Priority list: HDMI, SDI, anything else. +BMDVideoConnection pick_default_video_connection(IDeckLink *card, BMDDeckLinkAttributeID attribute_id, unsigned card_index); + +#endif // !defined(_DECKLINK_UTIL) diff --git a/nageru/defs.h b/nageru/defs.h new file mode 100644 index 0000000..7b8cc69 --- /dev/null +++ b/nageru/defs.h @@ -0,0 +1,55 @@ +#ifndef _DEFS_H +#define _DEFS_H + +#include + +// This flag is only supported in FFmpeg 3.3 and up, and we only require 3.1. +#if LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(57, 71, 100) +#define MUX_SKIP_TRAILER "+skip_trailer" +#else +#define MUX_SKIP_TRAILER "" +#endif + +#define OUTPUT_FREQUENCY 48000 // Currently needs to be exactly 48000, since bmusb outputs in that. +#define MAX_FPS 60 +#define FAKE_FPS 25 // Must be an integer. +#define MAX_VIDEO_CARDS 16 +#define MAX_ALSA_CARDS 16 +#define MAX_BUSES 256 // Audio buses. + +// For deinterlacing. See also comments on InputState. +#define FRAME_HISTORY_LENGTH 5 + +#define AUDIO_OUTPUT_CODEC_NAME "pcm_s32le" +#define DEFAULT_AUDIO_OUTPUT_BIT_RATE 0 +#define DEFAULT_X264_OUTPUT_BIT_RATE 4500 // 5 Mbit after making room for some audio and TCP overhead. + +#define LOCAL_DUMP_PREFIX "record-" +#define LOCAL_DUMP_SUFFIX ".nut" +#define DEFAULT_STREAM_MUX_NAME "nut" // Only for HTTP. Local dump guesses from LOCAL_DUMP_SUFFIX. +#define DEFAULT_HTTPD_PORT 9095 +#define MUX_OPTS { \ + /* Make seekable .mov files, and keep MP4 muxer from using unlimited amounts of memory. */ \ + { "movflags", "empty_moov+frag_keyframe+default_base_moof" MUX_SKIP_TRAILER }, \ + \ + /* Make for somewhat less bursty stream output when using .mov. */ \ + { "frag_duration", "125000" }, \ + \ + /* Keep nut muxer from using unlimited amounts of memory. */ \ + { "write_index", "0" } \ +} + +// In bytes. Beware, if too small, stream clients will start dropping data. +// For mov, you want this at 10MB or so (for the reason mentioned above), +// but for nut, there's no flushing, so such a large mux buffer would cause +// the output to be very uneven. +#define MUX_BUFFER_SIZE 10485760 + +// In number of frames. Comes in addition to any internal queues in x264 +// (frame threading, lookahead, etc.). +#define X264_QUEUE_LENGTH 50 + +#define X264_DEFAULT_PRESET "ultrafast" +#define X264_DEFAULT_TUNE "film" + +#endif // !defined(_DEFS_H) diff --git a/nageru/disk_space_estimator.cpp b/nageru/disk_space_estimator.cpp new file mode 100644 index 0000000..86e5e87 --- /dev/null +++ b/nageru/disk_space_estimator.cpp @@ -0,0 +1,64 @@ +#include "disk_space_estimator.h" + +#include +#include +#include +#include + +#include "metrics.h" +#include "timebase.h" + +DiskSpaceEstimator::DiskSpaceEstimator(DiskSpaceEstimator::callback_t callback) + : callback(callback) +{ + global_metrics.add("disk_free_bytes", &metric_disk_free_bytes, Metrics::TYPE_GAUGE); +} + +void DiskSpaceEstimator::report_write(const std::string &filename, uint64_t pts) +{ + if (filename != last_filename) { + last_filename = filename; + measure_points.clear(); + } + + // Reject points that are out-of-order (happens with B-frames). + if (!measure_points.empty() && pts < measure_points.back().pts) { + return; + } + + // Remove too old points. + while (measure_points.size() > 1 && measure_points.front().pts + window_length < pts) { + measure_points.pop_front(); + } + + struct stat st; + if (stat(filename.c_str(), &st) == -1) { + perror(filename.c_str()); + return; + } + + struct statfs fst; + if (statfs(filename.c_str(), &fst) == -1) { + perror(filename.c_str()); + return; + } + + off_t free_bytes = off_t(fst.f_bavail) * fst.f_frsize; + metric_disk_free_bytes = free_bytes; + + if (!measure_points.empty()) { + double bytes_per_second = double(st.st_size - measure_points.front().size) / + (pts - measure_points.front().pts) * TIMEBASE; + double seconds_left = free_bytes / bytes_per_second; + + // Only report every second, since updating the UI can be expensive. + if (last_pts_reported == 0 || pts - last_pts_reported >= TIMEBASE) { + callback(free_bytes, seconds_left); + last_pts_reported = pts; + } + } + + measure_points.push_back({ pts, st.st_size }); +} + +DiskSpaceEstimator *global_disk_space_estimator = nullptr; // Created in MainWindow::MainWindow(). diff --git a/nageru/disk_space_estimator.h b/nageru/disk_space_estimator.h new file mode 100644 index 0000000..73b392c --- /dev/null +++ b/nageru/disk_space_estimator.h @@ -0,0 +1,55 @@ +#ifndef _DISK_SPACE_ESTIMATOR_H +#define _DISK_SPACE_ESTIMATOR_H + +// A class responsible for measuring how much disk there is left when we +// store our video to disk, and how much recording time that equates to. +// It gets callbacks from the Mux writing the stream to disk (which also +// knows which filesystem the file is going to), makes its calculations, +// and calls back to the MainWindow, which shows it to the user. +// +// The bitrate is measured over a simple 30-second sliding window. + +#include +#include +#include +#include +#include +#include + +#include "timebase.h" + +class DiskSpaceEstimator +{ +public: + typedef std::function callback_t; + DiskSpaceEstimator(callback_t callback); + + // Report that a video frame with the given pts has just been written + // to the given file, so the estimator should stat the file and see + // by how much it grew since last time. Called by the Mux object + // responsible for writing to the stream on disk. + // + // If the filename changed since last time, the estimation is reset. + // is taken to be in TIMEBASE units (see timebase.h). + void report_write(const std::string &filename, uint64_t pts); + +private: + static constexpr int64_t window_length = 30 * TIMEBASE; + + callback_t callback; + std::string last_filename; + + struct MeasurePoint { + uint64_t pts; + off_t size; + }; + std::deque measure_points; + uint64_t last_pts_reported = 0; + + // Metrics. + std::atomic metric_disk_free_bytes{-1}; +}; + +extern DiskSpaceEstimator *global_disk_space_estimator; + +#endif // !defined(_DISK_SPACE_ESTIMATOR_H) diff --git a/nageru/display.ui b/nageru/display.ui new file mode 100644 index 0000000..a09294f --- /dev/null +++ b/nageru/display.ui @@ -0,0 +1,119 @@ + + + Display + + + + 0 + 0 + 606 + 433 + + + + Form + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + 0 + 1 + + + + true + + + QFrame::Box + + + QFrame::Plain + + + 0 + + + + 3 + + + 3 + + + 3 + + + 3 + + + + + + 0 + 0 + + + + + + + + + + + + + + 1 + 0 + + + + + 0 + 24 + + + + TextLabel + + + Qt::AlignCenter + + + + + + + Set WB + + + + + + + + + + GLWidget + QWidget +
glwidget.h
+ + + + + diff --git a/nageru/ebu_r128_proc.cc b/nageru/ebu_r128_proc.cc new file mode 100644 index 0000000..f062eaf --- /dev/null +++ b/nageru/ebu_r128_proc.cc @@ -0,0 +1,338 @@ +// ------------------------------------------------------------------------ +// +// Copyright (C) 2010-2011 Fons Adriaensen +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +// +// ------------------------------------------------------------------------ + + +#include "ebu_r128_proc.h" + +#include +#include + + +float Ebu_r128_hist::_bin_power [100] = { 0.0f }; +float Ebu_r128_proc::_chan_gain [5] = { 1.0f, 1.0f, 1.0f, 1.41f, 1.41f }; + + +Ebu_r128_hist::Ebu_r128_hist (void) +{ + _histc = new int [751]; + initstat (); + reset (); +} + + +Ebu_r128_hist::~Ebu_r128_hist (void) +{ + delete[] _histc; +} + + +void Ebu_r128_hist::reset (void) +{ + memset (_histc, 0, 751 * sizeof (float)); + _count = 0; + _error = 0; +} + + +void Ebu_r128_hist::initstat (void) +{ + int i; + + if (_bin_power [0]) return; + for (i = 0; i < 100; i++) + { + _bin_power [i] = powf (10.0f, i / 100.0f); + } +} + + +void Ebu_r128_hist::addpoint (float v) +{ + int k; + + k = (int) floorf (10 * v + 700.5f); + if (k < 0) return; + if (k > 750) + { + k = 750; + _error++; + } + _histc [k]++; + _count++; +} + + +float Ebu_r128_hist::integrate (int i) +{ + int j, k, n; + float s; + + j = i % 100; + n = 0; + s = 0; + while (i <= 750) + { + k = _histc [i++]; + n += k; + s += k * _bin_power [j++]; + if (j == 100) + { + j = 0; + s /= 10.0f; + } + } + return s / n; +} + + +void Ebu_r128_hist::calc_integ (float *vi, float *th) +{ + int k; + float s; + + if (_count < 50) + { + *vi = -200.0f; + return; + } + s = integrate (0); +// Original threshold was -8 dB below result of first integration +// if (th) *th = 10 * log10f (s) - 8.0f; +// k = (int)(floorf (100 * log10f (s) + 0.5f)) + 620; +// Threshold redefined to -10 dB below result of first integration + if (th) *th = 10 * log10f (s) - 10.0f; + k = (int)(floorf (100 * log10f (s) + 0.5f)) + 600; + if (k < 0) k = 0; + s = integrate (k); + *vi = 10 * log10f (s); +} + + +void Ebu_r128_hist::calc_range (float *v0, float *v1, float *th) +{ + int i, j, k, n; + float a, b, s; + + if (_count < 20) + { + *v0 = -200.0f; + *v1 = -200.0f; + return; + } + s = integrate (0); + if (th) *th = 10 * log10f (s) - 20.0f; + k = (int)(floorf (100 * log10f (s) + 0.5)) + 500; + if (k < 0) k = 0; + for (i = k, n = 0; i <= 750; i++) n += _histc [i]; + a = 0.10f * n; + b = 0.95f * n; + for (i = k, s = 0; s < a; i++) s += _histc [i]; + for (j = 750, s = n; s > b; j--) s -= _histc [j]; + *v0 = (i - 701) / 10.0f; + *v1 = (j - 699) / 10.0f; +} + + + + +Ebu_r128_proc::Ebu_r128_proc (void) +{ + reset (); +} + + +Ebu_r128_proc::~Ebu_r128_proc (void) +{ +} + + +void Ebu_r128_proc::init (int nchan, float fsamp) +{ + _nchan = nchan; + _fsamp = fsamp; + _fragm = (int) fsamp / 20; + detect_init (_fsamp); + reset (); +} + + +void Ebu_r128_proc::reset (void) +{ + _integr = false; + _frcnt = _fragm; + _frpwr = 1e-30f; + _wrind = 0; + _div1 = 0; + _div2 = 0; + _loudness_M = -200.0f; + _loudness_S = -200.0f; + memset (_power, 0, 64 * sizeof (float)); + integr_reset (); + detect_reset (); +} + + +void Ebu_r128_proc::integr_reset (void) +{ + _hist_M.reset (); + _hist_S.reset (); + _maxloudn_M = -200.0f; + _maxloudn_S = -200.0f; + _integrated = -200.0f; + _integ_thr = -200.0f; + _range_min = -200.0f; + _range_max = -200.0f; + _range_thr = -200.0f; + _div1 = _div2 = 0; +} + + +void Ebu_r128_proc::process (int nfram, float *input []) +{ + int i, k; + + for (i = 0; i < _nchan; i++) _ipp [i] = input [i]; + while (nfram) + { + k = (_frcnt < nfram) ? _frcnt : nfram; + _frpwr += detect_process (k); + _frcnt -= k; + if (_frcnt == 0) + { + _power [_wrind++] = _frpwr / _fragm; + _frcnt = _fragm; + _frpwr = 1e-30f; + _wrind &= 63; + _loudness_M = addfrags (8); + _loudness_S = addfrags (60); + if (_loudness_M > _maxloudn_M) _maxloudn_M = _loudness_M; + if (_loudness_S > _maxloudn_S) _maxloudn_S = _loudness_S; + if (_integr) + { + if (++_div1 == 2) + { + _hist_M.addpoint (_loudness_M); + _div1 = 0; + } + if (++_div2 == 10) + { + _hist_S.addpoint (_loudness_S); + _div2 = 0; + _hist_M.calc_integ (&_integrated, &_integ_thr); + _hist_S.calc_range (&_range_min, &_range_max, &_range_thr); + } + } + } + for (i = 0; i < _nchan; i++) _ipp [i] += k; + nfram -= k; + } +} + + +float Ebu_r128_proc::addfrags (int nfrag) +{ + int i, k; + float s; + + s = 0; + k = (_wrind - nfrag) & 63; + for (i = 0; i < nfrag; i++) s += _power [(i + k) & 63]; + return -0.6976f + 10 * log10f (s / nfrag); +} + + +void Ebu_r128_proc::detect_init (float fsamp) +{ + float a, b, c, d, r, u1, u2, w1, w2; + + r = 1 / tan (4712.3890f / fsamp); + w1 = r / 1.12201f; + w2 = r * 1.12201f; + u1 = u2 = 1.4085f + 210.0f / fsamp; + a = u1 * w1; + b = w1 * w1; + c = u2 * w2; + d = w2 * w2; + r = 1 + a + b; + _a0 = (1 + c + d) / r; + _a1 = (2 - 2 * d) / r; + _a2 = (1 - c + d) / r; + _b1 = (2 - 2 * b) / r; + _b2 = (1 - a + b) / r; + r = 48.0f / fsamp; + a = 4.9886075f * r; + b = 6.2298014f * r * r; + r = 1 + a + b; + a *= 2 / r; + b *= 4 / r; + _c3 = a + b; + _c4 = b; + r = 1.004995f / r; + _a0 *= r; + _a1 *= r; + _a2 *= r; +} + + +void Ebu_r128_proc::detect_reset (void) +{ + for (int i = 0; i < MAXCH; i++) _fst [i].reset (); +} + + +float Ebu_r128_proc::detect_process (int nfram) +{ + int i, j; + float si, sj; + float x, y, z1, z2, z3, z4; + float *p; + Ebu_r128_fst *S; + + si = 0; + for (i = 0, S = _fst; i < _nchan; i++, S++) + { + z1 = S->_z1; + z2 = S->_z2; + z3 = S->_z3; + z4 = S->_z4; + p = _ipp [i]; + sj = 0; + for (j = 0; j < nfram; j++) + { + x = p [j] - _b1 * z1 - _b2 * z2 + 1e-15f; + y = _a0 * x + _a1 * z1 + _a2 * z2 - _c3 * z3 - _c4 * z4; + z2 = z1; + z1 = x; + z4 += z3; + z3 += y; + sj += y * y; + } + if (_nchan == 1) si = 2 * sj; + else si += _chan_gain [i] * sj; + S->_z1 = z1; + S->_z2 = z2; + S->_z3 = z3; + S->_z4 = z4; + } + return si; +} + + + diff --git a/nageru/ebu_r128_proc.h b/nageru/ebu_r128_proc.h new file mode 100644 index 0000000..dbecfcb --- /dev/null +++ b/nageru/ebu_r128_proc.h @@ -0,0 +1,136 @@ +// ------------------------------------------------------------------------ +// +// Copyright (C) 2010-2011 Fons Adriaensen +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +// +// ------------------------------------------------------------------------ + + +#ifndef __EBU_R128_PROC_H +#define __EBU_R128_PROC_H + + +#define MAXCH 5 + + +class Ebu_r128_fst +{ +private: + + friend class Ebu_r128_proc; + + void reset (void) { _z1 = _z2 = _z3 = _z4 = 0; } + + float _z1, _z2, _z3, _z4; +}; + + +class Ebu_r128_hist +{ +private: + + Ebu_r128_hist (void); + ~Ebu_r128_hist (void); + + friend class Ebu_r128_proc; + + void reset (void); + void initstat (void); + void addpoint (float v); + float integrate (int ind); + void calc_integ (float *vi, float *th); + void calc_range (float *v0, float *v1, float *th); + + int *_histc; + int _count; + int _error; + + static float _bin_power [100]; +}; + + + +class Ebu_r128_proc +{ +public: + + Ebu_r128_proc (void); + ~Ebu_r128_proc (void); + + void init (int nchan, float fsamp); + void reset (void); + void process (int nfram, float *input []); + void integr_reset (void); + void integr_pause (void) { _integr = false; } + void integr_start (void) { _integr = true; } + + float loudness_M (void) const { return _loudness_M; } + float maxloudn_M (void) const { return _maxloudn_M; } + float loudness_S (void) const { return _loudness_S; } + float maxloudn_S (void) const { return _maxloudn_S; } + float integrated (void) const { return _integrated; } + float integ_thr (void) const { return _integ_thr; } + float range_min (void) const { return _range_min; } + float range_max (void) const { return _range_max; } + float range_thr (void) const { return _range_thr; } + + const int *histogram_M (void) const { return _hist_M._histc; } + const int *histogram_S (void) const { return _hist_S._histc; } + int hist_M_count (void) const { return _hist_M._count; } + int hist_S_count (void) const { return _hist_S._count; } + +private: + + float addfrags (int nfrag); + void detect_init (float fsamp); + void detect_reset (void); + float detect_process (int nfram); + + bool _integr; // Integration on/off. + int _nchan; // Number of channels, 2 or 5. + float _fsamp; // Sample rate. + int _fragm; // Fragmenst size, 1/20 second. + int _frcnt; // Number of samples remaining in current fragment. + float _frpwr; // Power accumulated for current fragment. + float _power [64]; // Array of fragment powers. + int _wrind; // Write index into _frpwr + int _div1; // M period counter, 200 ms; + int _div2; // S period counter, 1s; + float _loudness_M; + float _maxloudn_M; + float _loudness_S; + float _maxloudn_S; + float _integrated; + float _integ_thr; + float _range_min; + float _range_max; + float _range_thr; + + // Filter coefficients and states. + float _a0, _a1, _a2; + float _b1, _b2; + float _c3, _c4; + float *_ipp [MAXCH]; + Ebu_r128_fst _fst [MAXCH]; + Ebu_r128_hist _hist_M; + Ebu_r128_hist _hist_S; + + // Default channel gains. + static float _chan_gain [5]; +}; + + +#endif diff --git a/nageru/ellipsis_label.h b/nageru/ellipsis_label.h new file mode 100644 index 0000000..bec3799 --- /dev/null +++ b/nageru/ellipsis_label.h @@ -0,0 +1,35 @@ +#ifndef _ELLIPSIS_LABEL_H +#define _ELLIPSIS_LABEL_H 1 + +#include + +class EllipsisLabel : public QLabel { + Q_OBJECT + +public: + EllipsisLabel(QWidget *parent) : QLabel(parent) {} + + void setFullText(const QString &s) + { + full_text = s; + updateEllipsisText(); + } + +protected: + void resizeEvent(QResizeEvent *event) override + { + QLabel::resizeEvent(event); + updateEllipsisText(); + } + +private: + void updateEllipsisText() + { + QFontMetrics metrics(this->font()); + this->setText(metrics.elidedText(full_text, Qt::ElideRight, this->width())); + } + + QString full_text; +}; + +#endif // !defined(_ELLIPSIS_LABEL_H) diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp new file mode 100644 index 0000000..7bd9278 --- /dev/null +++ b/nageru/ffmpeg_capture.cpp @@ -0,0 +1,888 @@ +#include "ffmpeg_capture.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +} + +#include +#include +#include +#include + +#include "bmusb/bmusb.h" +#include "ffmpeg_raii.h" +#include "ffmpeg_util.h" +#include "flags.h" +#include "image_input.h" +#include "ref_counted_frame.h" +#include "timebase.h" + +#define FRAME_SIZE (8 << 20) // 8 MB. + +using namespace std; +using namespace std::chrono; +using namespace bmusb; +using namespace movit; + +namespace { + +steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate) +{ + const duration pts((frame_pts - pts_origin) * double(video_timebase.num) / double(video_timebase.den)); + return origin + duration_cast(pts / rate); +} + +bool changed_since(const std::string &pathname, const timespec &ts) +{ + if (ts.tv_sec < 0) { + return false; + } + struct stat buf; + if (stat(pathname.c_str(), &buf) != 0) { + fprintf(stderr, "%s: Couldn't check for new version, leaving the old in place.\n", pathname.c_str()); + return false; + } + return (buf.st_mtim.tv_sec != ts.tv_sec || buf.st_mtim.tv_nsec != ts.tv_nsec); +} + +bool is_full_range(const AVPixFmtDescriptor *desc) +{ + // This is horrible, but there's no better way that I know of. + return (strchr(desc->name, 'j') != nullptr); +} + +AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst_format_type) +{ + if (dst_format_type == bmusb::PixelFormat_8BitBGRA) { + return AV_PIX_FMT_BGRA; + } + if (dst_format_type == FFmpegCapture::PixelFormat_NV12) { + return AV_PIX_FMT_NV12; + } + + assert(dst_format_type == bmusb::PixelFormat_8BitYCbCrPlanar); + + // If this is a non-Y'CbCr format, just convert to 4:4:4 Y'CbCr + // and be done with it. It's too strange to spend a lot of time on. + // (Let's hope there's no alpha.) + const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get(src_format); + if (src_desc == nullptr || + src_desc->nb_components != 3 || + (src_desc->flags & AV_PIX_FMT_FLAG_RGB)) { + return AV_PIX_FMT_YUV444P; + } + + // The best for us would be Cb and Cr together if possible, + // but FFmpeg doesn't support that except in the special case of + // NV12, so we need to go to planar even for the case of NV12. + // Thus, look for the closest (but no worse) 8-bit planar Y'CbCr format + // that matches in color range. (This will also include the case of + // the source format already being acceptable.) + bool src_full_range = is_full_range(src_desc); + const char *best_format = "yuv444p"; + unsigned best_score = numeric_limits::max(); + for (const AVPixFmtDescriptor *desc = av_pix_fmt_desc_next(nullptr); + desc; + desc = av_pix_fmt_desc_next(desc)) { + // Find planar Y'CbCr formats only. + if (desc->nb_components != 3) continue; + if (desc->flags & AV_PIX_FMT_FLAG_RGB) continue; + if (!(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) continue; + if (desc->comp[0].plane != 0 || + desc->comp[1].plane != 1 || + desc->comp[2].plane != 2) continue; + + // 8-bit formats only. + if (desc->flags & AV_PIX_FMT_FLAG_BE) continue; + if (desc->comp[0].depth != 8) continue; + + // Same or better chroma resolution only. + int chroma_w_diff = desc->log2_chroma_w - src_desc->log2_chroma_w; + int chroma_h_diff = desc->log2_chroma_h - src_desc->log2_chroma_h; + if (chroma_w_diff < 0 || chroma_h_diff < 0) + continue; + + // Matching full/limited range only. + if (is_full_range(desc) != src_full_range) + continue; + + // Pick something with as little excess chroma resolution as possible. + unsigned score = (1 << (chroma_w_diff)) << chroma_h_diff; + if (score < best_score) { + best_score = score; + best_format = desc->name; + } + } + return av_get_pix_fmt(best_format); +} + +YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame) +{ + YCbCrFormat format; + AVColorSpace colorspace = av_frame_get_colorspace(frame); + switch (colorspace) { + case AVCOL_SPC_BT709: + format.luma_coefficients = YCBCR_REC_709; + break; + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + case AVCOL_SPC_SMPTE240M: + format.luma_coefficients = YCBCR_REC_601; + break; + case AVCOL_SPC_BT2020_NCL: + format.luma_coefficients = YCBCR_REC_2020; + break; + case AVCOL_SPC_UNSPECIFIED: + format.luma_coefficients = (frame->height >= 720 ? YCBCR_REC_709 : YCBCR_REC_601); + break; + default: + fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n", + colorspace); + format.luma_coefficients = YCBCR_REC_709; + break; + } + + format.full_range = is_full_range(desc); + format.num_levels = 1 << desc->comp[0].depth; + format.chroma_subsampling_x = 1 << desc->log2_chroma_w; + format.chroma_subsampling_y = 1 << desc->log2_chroma_h; + + switch (frame->chroma_location) { + case AVCHROMA_LOC_LEFT: + format.cb_x_position = 0.0; + format.cb_y_position = 0.5; + break; + case AVCHROMA_LOC_CENTER: + format.cb_x_position = 0.5; + format.cb_y_position = 0.5; + break; + case AVCHROMA_LOC_TOPLEFT: + format.cb_x_position = 0.0; + format.cb_y_position = 0.0; + break; + case AVCHROMA_LOC_TOP: + format.cb_x_position = 0.5; + format.cb_y_position = 0.0; + break; + case AVCHROMA_LOC_BOTTOMLEFT: + format.cb_x_position = 0.0; + format.cb_y_position = 1.0; + break; + case AVCHROMA_LOC_BOTTOM: + format.cb_x_position = 0.5; + format.cb_y_position = 1.0; + break; + default: + fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing Rec. 709.\n", + frame->chroma_location); + format.cb_x_position = 0.5; + format.cb_y_position = 0.5; + break; + } + + format.cr_x_position = format.cb_x_position; + format.cr_y_position = format.cb_y_position; + return format; +} + +} // namespace + +FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height) + : filename(filename), width(width), height(height), video_timebase{1, 1} +{ + description = "Video: " + filename; + + last_frame = steady_clock::now(); + + avformat_network_init(); // In case someone wants this. +} + +FFmpegCapture::~FFmpegCapture() +{ + if (has_dequeue_callbacks) { + dequeue_cleanup_callback(); + } + avresample_free(&resampler); +} + +void FFmpegCapture::configure_card() +{ + if (video_frame_allocator == nullptr) { + owned_video_frame_allocator.reset(new MallocFrameAllocator(FRAME_SIZE, NUM_QUEUED_VIDEO_FRAMES)); + set_video_frame_allocator(owned_video_frame_allocator.get()); + } + if (audio_frame_allocator == nullptr) { + // Audio can come out in pretty large chunks, so increase from the default 1 MB. + owned_audio_frame_allocator.reset(new MallocFrameAllocator(1 << 20, NUM_QUEUED_AUDIO_FRAMES)); + set_audio_frame_allocator(owned_audio_frame_allocator.get()); + } +} + +void FFmpegCapture::start_bm_capture() +{ + if (running) { + return; + } + running = true; + producer_thread_should_quit.unquit(); + producer_thread = thread(&FFmpegCapture::producer_thread_func, this); +} + +void FFmpegCapture::stop_dequeue_thread() +{ + if (!running) { + return; + } + running = false; + producer_thread_should_quit.quit(); + producer_thread.join(); +} + +std::map FFmpegCapture::get_available_video_modes() const +{ + // Note: This will never really be shown in the UI. + VideoMode mode; + + char buf[256]; + snprintf(buf, sizeof(buf), "%ux%u", width, height); + mode.name = buf; + + mode.autodetect = false; + mode.width = width; + mode.height = height; + mode.frame_rate_num = 60; + mode.frame_rate_den = 1; + mode.interlaced = false; + + return {{ 0, mode }}; +} + +void FFmpegCapture::producer_thread_func() +{ + char thread_name[16]; + snprintf(thread_name, sizeof(thread_name), "FFmpeg_C_%d", card_index); + pthread_setname_np(pthread_self(), thread_name); + + while (!producer_thread_should_quit.should_quit()) { + string filename_copy; + { + lock_guard lock(filename_mu); + filename_copy = filename; + } + + string pathname = search_for_file(filename_copy); + if (pathname.empty()) { + fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str()); + send_disconnected_frame(); + producer_thread_should_quit.sleep_for(seconds(1)); + continue; + } + should_interrupt = false; + if (!play_video(pathname)) { + // Error. + fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str()); + send_disconnected_frame(); + producer_thread_should_quit.sleep_for(seconds(1)); + continue; + } + + // Probably just EOF, will exit the loop above on next test. + } + + if (has_dequeue_callbacks) { + dequeue_cleanup_callback(); + has_dequeue_callbacks = false; + } +} + +void FFmpegCapture::send_disconnected_frame() +{ + // Send an empty frame to signal that we have no signal anymore. + FrameAllocator::Frame video_frame = video_frame_allocator->alloc_frame(); + if (video_frame.data) { + VideoFormat video_format; + video_format.width = width; + video_format.height = height; + video_format.frame_rate_nom = 60; + video_format.frame_rate_den = 1; + video_format.is_connected = false; + if (pixel_format == bmusb::PixelFormat_8BitBGRA) { + video_format.stride = width * 4; + video_frame.len = width * height * 4; + memset(video_frame.data, 0, video_frame.len); + } else { + video_format.stride = width; + current_frame_ycbcr_format.luma_coefficients = YCBCR_REC_709; + current_frame_ycbcr_format.full_range = true; + current_frame_ycbcr_format.num_levels = 256; + current_frame_ycbcr_format.chroma_subsampling_x = 2; + current_frame_ycbcr_format.chroma_subsampling_y = 2; + current_frame_ycbcr_format.cb_x_position = 0.0f; + current_frame_ycbcr_format.cb_y_position = 0.0f; + current_frame_ycbcr_format.cr_x_position = 0.0f; + current_frame_ycbcr_format.cr_y_position = 0.0f; + video_frame.len = width * height * 2; + memset(video_frame.data, 0, width * height); + memset(video_frame.data + width * height, 128, width * height); // Valid for both NV12 and planar. + } + + frame_callback(-1, AVRational{1, TIMEBASE}, -1, AVRational{1, TIMEBASE}, timecode++, + video_frame, /*video_offset=*/0, video_format, + FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat()); + last_frame_was_connected = false; + } +} + +bool FFmpegCapture::play_video(const string &pathname) +{ + // Note: Call before open, not after; otherwise, there's a race. + // (There is now, too, but it tips the correct way. We could use fstat() + // if we had the file descriptor.) + timespec last_modified; + struct stat buf; + if (stat(pathname.c_str(), &buf) != 0) { + // Probably some sort of protocol, so can't stat. + last_modified.tv_sec = -1; + } else { + last_modified = buf.st_mtim; + } + + AVDictionary *opts = nullptr; + av_dict_set(&opts, "fflags", "nobuffer", 0); + + auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, &opts, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); + if (format_ctx == nullptr) { + fprintf(stderr, "%s: Error opening file\n", pathname.c_str()); + return false; + } + + if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) { + fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str()); + return false; + } + + int video_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_VIDEO); + if (video_stream_index == -1) { + fprintf(stderr, "%s: No video stream found\n", pathname.c_str()); + return false; + } + + int audio_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_AUDIO); + + // Open video decoder. + const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar; + AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id); + video_timebase = format_ctx->streams[video_stream_index]->time_base; + AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr); + if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) { + fprintf(stderr, "%s: Cannot fill video codec parameters\n", pathname.c_str()); + return false; + } + if (video_codec == nullptr) { + fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str()); + return false; + } + if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) { + fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str()); + return false; + } + unique_ptr video_codec_ctx_cleanup( + video_codec_ctx.get(), avcodec_close); + + // Open audio decoder, if we have audio. + AVCodecContextWithDeleter audio_codec_ctx; + if (audio_stream_index != -1) { + audio_codec_ctx = avcodec_alloc_context3_unique(nullptr); + const AVCodecParameters *audio_codecpar = format_ctx->streams[audio_stream_index]->codecpar; + audio_timebase = format_ctx->streams[audio_stream_index]->time_base; + if (avcodec_parameters_to_context(audio_codec_ctx.get(), audio_codecpar) < 0) { + fprintf(stderr, "%s: Cannot fill audio codec parameters\n", pathname.c_str()); + return false; + } + AVCodec *audio_codec = avcodec_find_decoder(audio_codecpar->codec_id); + if (audio_codec == nullptr) { + fprintf(stderr, "%s: Cannot find audio decoder\n", pathname.c_str()); + return false; + } + if (avcodec_open2(audio_codec_ctx.get(), audio_codec, nullptr) < 0) { + fprintf(stderr, "%s: Cannot open audio decoder\n", pathname.c_str()); + return false; + } + } + unique_ptr audio_codec_ctx_cleanup( + audio_codec_ctx.get(), avcodec_close); + + internal_rewind(); + + // Main loop. + bool first_frame = true; + while (!producer_thread_should_quit.should_quit()) { + if (process_queued_commands(format_ctx.get(), pathname, last_modified, /*rewound=*/nullptr)) { + return true; + } + UniqueFrame audio_frame = audio_frame_allocator->alloc_frame(); + AudioFormat audio_format; + + int64_t audio_pts; + bool error; + AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(), + pathname, video_stream_index, audio_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error); + if (error) { + return false; + } + if (frame == nullptr) { + // EOF. Loop back to the start if we can. + if (av_seek_frame(format_ctx.get(), /*stream_index=*/-1, /*timestamp=*/0, /*flags=*/0) < 0) { + fprintf(stderr, "%s: Rewind failed, not looping.\n", pathname.c_str()); + return true; + } + if (video_codec_ctx != nullptr) { + avcodec_flush_buffers(video_codec_ctx.get()); + } + if (audio_codec_ctx != nullptr) { + avcodec_flush_buffers(audio_codec_ctx.get()); + } + // If the file has changed since last time, return to get it reloaded. + // Note that depending on how you move the file into place, you might + // end up corrupting the one you're already playing, so this path + // might not trigger. + if (changed_since(pathname, last_modified)) { + return true; + } + internal_rewind(); + continue; + } + + VideoFormat video_format = construct_video_format(frame.get(), video_timebase); + UniqueFrame video_frame = make_video_frame(frame.get(), pathname, &error); + if (error) { + return false; + } + + for ( ;; ) { + if (last_pts == 0 && pts_origin == 0) { + pts_origin = frame->pts; + } + next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate); + if (first_frame && last_frame_was_connected) { + // If reconnect took more than one second, this is probably a live feed, + // and we should reset the resampler. (Or the rate is really, really low, + // in which case a reset on the first frame is fine anyway.) + if (duration(next_frame_start - last_frame).count() >= 1.0) { + last_frame_was_connected = false; + } + } + video_frame->received_timestamp = next_frame_start; + + // The easiest way to get all the rate conversions etc. right is to move the + // audio PTS into the video PTS timebase and go from there. (We'll get some + // rounding issues, but they should not be a big problem.) + int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase); + audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate); + + if (audio_frame->len != 0) { + // The received timestamps in Nageru are measured after we've just received the frame. + // However, pts (especially audio pts) is at the _beginning_ of the frame. + // If we have locked audio, the distinction doesn't really matter, as pts is + // on a relative scale and a fixed offset is fine. But if we don't, we will have + // a different number of samples each time, which will cause huge audio jitter + // and throw off the resampler. + // + // In a sense, we should have compensated by adding the frame and audio lengths + // to video_frame->received_timestamp and audio_frame->received_timestamp respectively, + // but that would mean extra waiting in sleep_until(). All we need is that they + // are correct relative to each other, though (and to the other frames we send), + // so just align the end of the audio frame, and we're fine. + size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels; + double offset = double(num_samples) / OUTPUT_FREQUENCY - + double(video_format.frame_rate_den) / video_format.frame_rate_nom; + audio_frame->received_timestamp += duration_cast(duration(offset)); + } + + steady_clock::time_point now = steady_clock::now(); + if (duration(now - next_frame_start).count() >= 0.1) { + // If we don't have enough CPU to keep up, or if we have a live stream + // where the initial origin was somehow wrong, we could be behind indefinitely. + // In particular, this will give the audio resampler problems as it tries + // to speed up to reduce the delay, hitting the low end of the buffer every time. + fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n", + pathname.c_str(), + 1e3 * duration(now - next_frame_start).count()); + pts_origin = frame->pts; + start = next_frame_start = now; + timecode += MAX_FPS * 2 + 1; + } + bool finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start); + if (finished_wakeup) { + if (audio_frame->len > 0) { + assert(audio_pts != -1); + } + if (!last_frame_was_connected) { + // We're recovering from an error (or really slow load, see above). + // Make sure to get the audio resampler reset. (This is a hack; + // ideally, the frame callback should just accept a way to signal + // audio discontinuity.) + timecode += MAX_FPS * 2 + 1; + } + frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++, + video_frame.get_and_release(), 0, video_format, + audio_frame.get_and_release(), 0, audio_format); + first_frame = false; + last_frame = steady_clock::now(); + last_frame_was_connected = true; + break; + } else { + if (producer_thread_should_quit.should_quit()) break; + + bool rewound = false; + if (process_queued_commands(format_ctx.get(), pathname, last_modified, &rewound)) { + return true; + } + // If we just rewound, drop this frame on the floor and be done. + if (rewound) { + break; + } + // OK, we didn't, so probably a rate change. Recalculate next_frame_start, + // but if it's now in the past, we'll reset the origin, so that we don't + // generate a huge backlog of frames that we need to run through quickly. + next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate); + steady_clock::time_point now = steady_clock::now(); + if (next_frame_start < now) { + pts_origin = frame->pts; + start = next_frame_start = now; + } + } + } + last_pts = frame->pts; + } + return true; +} + +void FFmpegCapture::internal_rewind() +{ + pts_origin = last_pts = 0; + start = next_frame_start = steady_clock::now(); +} + +bool FFmpegCapture::process_queued_commands(AVFormatContext *format_ctx, const std::string &pathname, timespec last_modified, bool *rewound) +{ + // Process any queued commands from other threads. + vector commands; + { + lock_guard lock(queue_mu); + swap(commands, command_queue); + } + for (const QueuedCommand &cmd : commands) { + switch (cmd.command) { + case QueuedCommand::REWIND: + if (av_seek_frame(format_ctx, /*stream_index=*/-1, /*timestamp=*/0, /*flags=*/0) < 0) { + fprintf(stderr, "%s: Rewind failed, stopping play.\n", pathname.c_str()); + } + // If the file has changed since last time, return to get it reloaded. + // Note that depending on how you move the file into place, you might + // end up corrupting the one you're already playing, so this path + // might not trigger. + if (changed_since(pathname, last_modified)) { + return true; + } + internal_rewind(); + if (rewound != nullptr) { + *rewound = true; + } + break; + + case QueuedCommand::CHANGE_RATE: + // Change the origin to the last played frame. + start = compute_frame_start(last_pts, pts_origin, video_timebase, start, rate); + pts_origin = last_pts; + rate = cmd.new_rate; + break; + } + } + return false; +} + +namespace { + +} // namespace + +AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx, + const std::string &pathname, int video_stream_index, int audio_stream_index, + FrameAllocator::Frame *audio_frame, AudioFormat *audio_format, int64_t *audio_pts, bool *error) +{ + *error = false; + + // Read packets until we have a frame or there are none left. + bool frame_finished = false; + AVFrameWithDeleter audio_avframe = av_frame_alloc_unique(); + AVFrameWithDeleter video_avframe = av_frame_alloc_unique(); + bool eof = false; + *audio_pts = -1; + bool has_audio = false; + do { + AVPacket pkt; + unique_ptr pkt_cleanup( + &pkt, av_packet_unref); + av_init_packet(&pkt); + pkt.data = nullptr; + pkt.size = 0; + if (av_read_frame(format_ctx, &pkt) == 0) { + if (pkt.stream_index == audio_stream_index && audio_callback != nullptr) { + audio_callback(&pkt, format_ctx->streams[audio_stream_index]->time_base); + } + if (pkt.stream_index == video_stream_index) { + if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) { + fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str()); + *error = true; + return AVFrameWithDeleter(nullptr); + } + } else if (pkt.stream_index == audio_stream_index) { + has_audio = true; + if (avcodec_send_packet(audio_codec_ctx, &pkt) < 0) { + fprintf(stderr, "%s: Cannot send packet to audio codec.\n", pathname.c_str()); + *error = true; + return AVFrameWithDeleter(nullptr); + } + } + } else { + eof = true; // Or error, but ignore that for the time being. + } + + // Decode audio, if any. + if (has_audio) { + for ( ;; ) { + int err = avcodec_receive_frame(audio_codec_ctx, audio_avframe.get()); + if (err == 0) { + if (*audio_pts == -1) { + *audio_pts = audio_avframe->pts; + } + convert_audio(audio_avframe.get(), audio_frame, audio_format); + } else if (err == AVERROR(EAGAIN)) { + break; + } else { + fprintf(stderr, "%s: Cannot receive frame from audio codec.\n", pathname.c_str()); + *error = true; + return AVFrameWithDeleter(nullptr); + } + } + } + + // Decode video, if we have a frame. + int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get()); + if (err == 0) { + frame_finished = true; + break; + } else if (err != AVERROR(EAGAIN)) { + fprintf(stderr, "%s: Cannot receive frame from video codec.\n", pathname.c_str()); + *error = true; + return AVFrameWithDeleter(nullptr); + } + } while (!eof); + + if (frame_finished) + return video_avframe; + else + return AVFrameWithDeleter(nullptr); +} + +void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator::Frame *audio_frame, AudioFormat *audio_format) +{ + // Decide on a format. If there already is one in this audio frame, + // we're pretty much forced to use it. If not, we try to find an exact match. + // If that still doesn't work, we default to 32-bit signed chunked + // (float would be nice, but there's really no way to signal that yet). + AVSampleFormat dst_format; + if (audio_format->bits_per_sample == 0) { + switch (audio_avframe->format) { + case AV_SAMPLE_FMT_S16: + case AV_SAMPLE_FMT_S16P: + audio_format->bits_per_sample = 16; + dst_format = AV_SAMPLE_FMT_S16; + break; + case AV_SAMPLE_FMT_S32: + case AV_SAMPLE_FMT_S32P: + default: + audio_format->bits_per_sample = 32; + dst_format = AV_SAMPLE_FMT_S32; + break; + } + } else if (audio_format->bits_per_sample == 16) { + dst_format = AV_SAMPLE_FMT_S16; + } else if (audio_format->bits_per_sample == 32) { + dst_format = AV_SAMPLE_FMT_S32; + } else { + assert(false); + } + audio_format->num_channels = 2; + + int64_t channel_layout = audio_avframe->channel_layout; + if (channel_layout == 0) { + channel_layout = av_get_default_channel_layout(audio_avframe->channels); + } + + if (resampler == nullptr || + audio_avframe->format != last_src_format || + dst_format != last_dst_format || + channel_layout != last_channel_layout || + av_frame_get_sample_rate(audio_avframe) != last_sample_rate) { + avresample_free(&resampler); + resampler = avresample_alloc_context(); + if (resampler == nullptr) { + fprintf(stderr, "Allocating resampler failed.\n"); + exit(1); + } + + av_opt_set_int(resampler, "in_channel_layout", channel_layout, 0); + av_opt_set_int(resampler, "out_channel_layout", AV_CH_LAYOUT_STEREO_DOWNMIX, 0); + av_opt_set_int(resampler, "in_sample_rate", av_frame_get_sample_rate(audio_avframe), 0); + av_opt_set_int(resampler, "out_sample_rate", OUTPUT_FREQUENCY, 0); + av_opt_set_int(resampler, "in_sample_fmt", audio_avframe->format, 0); + av_opt_set_int(resampler, "out_sample_fmt", dst_format, 0); + + if (avresample_open(resampler) < 0) { + fprintf(stderr, "Could not open resample context.\n"); + exit(1); + } + + last_src_format = AVSampleFormat(audio_avframe->format); + last_dst_format = dst_format; + last_channel_layout = channel_layout; + last_sample_rate = av_frame_get_sample_rate(audio_avframe); + } + + size_t bytes_per_sample = (audio_format->bits_per_sample / 8) * 2; + size_t num_samples_room = (audio_frame->size - audio_frame->len) / bytes_per_sample; + + uint8_t *data = audio_frame->data + audio_frame->len; + int out_samples = avresample_convert(resampler, &data, 0, num_samples_room, + const_cast(audio_avframe->data), audio_avframe->linesize[0], audio_avframe->nb_samples); + if (out_samples < 0) { + fprintf(stderr, "Audio conversion failed.\n"); + exit(1); + } + + audio_frame->len += out_samples * bytes_per_sample; +} + +VideoFormat FFmpegCapture::construct_video_format(const AVFrame *frame, AVRational video_timebase) +{ + VideoFormat video_format; + video_format.width = width; + video_format.height = height; + if (pixel_format == bmusb::PixelFormat_8BitBGRA) { + video_format.stride = width * 4; + } else if (pixel_format == FFmpegCapture::PixelFormat_NV12) { + video_format.stride = width; + } else { + assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); + video_format.stride = width; + } + video_format.frame_rate_nom = video_timebase.den; + video_format.frame_rate_den = av_frame_get_pkt_duration(frame) * video_timebase.num; + if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) { + // Invalid frame rate. + video_format.frame_rate_nom = 60; + video_format.frame_rate_den = 1; + } + video_format.has_signal = true; + video_format.is_connected = true; + return video_format; +} + +UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string &pathname, bool *error) +{ + *error = false; + + UniqueFrame video_frame(video_frame_allocator->alloc_frame()); + if (video_frame->data == nullptr) { + return video_frame; + } + + if (sws_ctx == nullptr || + sws_last_width != frame->width || + sws_last_height != frame->height || + sws_last_src_format != frame->format) { + sws_dst_format = decide_dst_format(AVPixelFormat(frame->format), pixel_format); + sws_ctx.reset( + sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format), + width, height, sws_dst_format, + SWS_BICUBIC, nullptr, nullptr, nullptr)); + sws_last_width = frame->width; + sws_last_height = frame->height; + sws_last_src_format = frame->format; + } + if (sws_ctx == nullptr) { + fprintf(stderr, "%s: Could not create scaler context\n", pathname.c_str()); + *error = true; + return video_frame; + } + + uint8_t *pic_data[4] = { nullptr, nullptr, nullptr, nullptr }; + int linesizes[4] = { 0, 0, 0, 0 }; + if (pixel_format == bmusb::PixelFormat_8BitBGRA) { + pic_data[0] = video_frame->data; + linesizes[0] = width * 4; + video_frame->len = (width * 4) * height; + } else if (pixel_format == PixelFormat_NV12) { + pic_data[0] = video_frame->data; + linesizes[0] = width; + + pic_data[1] = pic_data[0] + width * height; + linesizes[1] = width; + + video_frame->len = (width * 2) * height; + + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame); + } else { + assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); + + int chroma_width = AV_CEIL_RSHIFT(int(width), desc->log2_chroma_w); + int chroma_height = AV_CEIL_RSHIFT(int(height), desc->log2_chroma_h); + + pic_data[0] = video_frame->data; + linesizes[0] = width; + + pic_data[1] = pic_data[0] + width * height; + linesizes[1] = chroma_width; + + pic_data[2] = pic_data[1] + chroma_width * chroma_height; + linesizes[2] = chroma_width; + + video_frame->len = width * height + 2 * chroma_width * chroma_height; + + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame); + } + sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes); + + return video_frame; +} + +int FFmpegCapture::interrupt_cb_thunk(void *unique) +{ + return reinterpret_cast(unique)->interrupt_cb(); +} + +int FFmpegCapture::interrupt_cb() +{ + return should_interrupt.load(); +} diff --git a/nageru/ffmpeg_capture.h b/nageru/ffmpeg_capture.h new file mode 100644 index 0000000..8a513df --- /dev/null +++ b/nageru/ffmpeg_capture.h @@ -0,0 +1,280 @@ +#ifndef _FFMPEG_CAPTURE_H +#define _FFMPEG_CAPTURE_H 1 + +// FFmpegCapture looks much like a capture card, but the frames it spits out +// come from a video in real time, looping. Because it decodes the video using +// FFmpeg (thus the name), this means it can handle a very wide array of video +// formats, and also things like network streaming and V4L capture, but it is +// also significantly less integrated and optimized than the regular capture +// cards. In particular, the frames are always scaled and converted to 8-bit +// RGBA on the CPU before being sent on to the GPU. +// +// Since we don't really know much about the video when building the chains, +// there are some limitations. In particular, frames are always assumed to be +// sRGB even if the video container says something else. We could probably +// try to load the video on startup and pick out the parameters at that point, +// but it would require some more plumbing, and it would also fail if the file +// changes parameters midway, which is allowed in some formats. +// +// You can get out the audio either as decoded or in raw form (Kaeru uses this). + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +extern "C" { +#include +#include +#include +#include +} + +#include "bmusb/bmusb.h" +#include "ffmpeg_raii.h" +#include "ref_counted_frame.h" +#include "quittable_sleeper.h" + +struct AVFormatContext; +struct AVFrame; +struct AVRational; +struct AVPacket; + +class FFmpegCapture : public bmusb::CaptureInterface +{ +public: + FFmpegCapture(const std::string &filename, unsigned width, unsigned height); + ~FFmpegCapture(); + + void set_card_index(int card_index) + { + this->card_index = card_index; + } + + int get_card_index() const + { + return card_index; + } + + void rewind() + { + std::lock_guard lock(queue_mu); + command_queue.push_back(QueuedCommand { QueuedCommand::REWIND }); + producer_thread_should_quit.wakeup(); + } + + void change_rate(double new_rate) + { + std::lock_guard lock(queue_mu); + command_queue.push_back(QueuedCommand { QueuedCommand::CHANGE_RATE, new_rate }); + producer_thread_should_quit.wakeup(); + } + + std::string get_filename() const + { + std::lock_guard lock(filename_mu); + return filename; + } + + void change_filename(const std::string &new_filename) + { + std::lock_guard lock(filename_mu); + filename = new_filename; + should_interrupt = true; + } + + // Will stop the stream even if it's hung on blocking I/O. + void disconnect() + { + should_interrupt = true; + } + + // CaptureInterface. + void set_video_frame_allocator(bmusb::FrameAllocator *allocator) override + { + video_frame_allocator = allocator; + if (owned_video_frame_allocator.get() != allocator) { + owned_video_frame_allocator.reset(); + } + } + + bmusb::FrameAllocator *get_video_frame_allocator() override + { + return video_frame_allocator; + } + + // Does not take ownership. + void set_audio_frame_allocator(bmusb::FrameAllocator *allocator) override + { + audio_frame_allocator = allocator; + if (owned_audio_frame_allocator.get() != allocator) { + owned_audio_frame_allocator.reset(); + } + } + + bmusb::FrameAllocator *get_audio_frame_allocator() override + { + return audio_frame_allocator; + } + + // FFmpegCapture-specific overload of set_frame_callback that also gives + // the raw original pts from the video. Negative pts means a dummy frame. + typedef std::function + frame_callback_t; + void set_frame_callback(frame_callback_t callback) + { + frame_callback = callback; + } + + void set_frame_callback(bmusb::frame_callback_t callback) override + { + frame_callback = std::bind( + callback, + std::placeholders::_5, + std::placeholders::_6, + std::placeholders::_7, + std::placeholders::_8, + std::placeholders::_9, + std::placeholders::_10, + std::placeholders::_11); + } + + // FFmpegCapture-specific callback that gives the raw audio. + typedef std::function audio_callback_t; + void set_audio_callback(audio_callback_t callback) + { + audio_callback = callback; + } + + // Used to get precise information about the Y'CbCr format used + // for a given frame. Only valid to call during the frame callback, + // and only when receiving a frame with pixel format PixelFormat_8BitYCbCrPlanar. + movit::YCbCrFormat get_current_frame_ycbcr_format() const + { + return current_frame_ycbcr_format; + } + + void set_dequeue_thread_callbacks(std::function init, std::function cleanup) override + { + dequeue_init_callback = init; + dequeue_cleanup_callback = cleanup; + has_dequeue_callbacks = true; + } + + std::string get_description() const override + { + return description; + } + + void configure_card() override; + void start_bm_capture() override; + void stop_dequeue_thread() override; + bool get_disconnected() const override { return false; } // We never unplug. + + std::map get_available_video_modes() const override; + void set_video_mode(uint32_t video_mode_id) override {} // Ignore. + uint32_t get_current_video_mode() const override { return 0; } + + static constexpr bmusb::PixelFormat PixelFormat_NV12 = static_cast(100); // In the private range. + std::set get_available_pixel_formats() const override { + return std::set{ bmusb::PixelFormat_8BitBGRA, bmusb::PixelFormat_8BitYCbCrPlanar, PixelFormat_NV12 }; + } + void set_pixel_format(bmusb::PixelFormat pixel_format) override { + this->pixel_format = pixel_format; + } + bmusb::PixelFormat get_current_pixel_format() const override { + return pixel_format; + } + + std::map get_available_video_inputs() const override { + return { { 0, "Auto" } }; } + void set_video_input(uint32_t video_input_id) override {} // Ignore. + uint32_t get_current_video_input() const override { return 0; } + + std::map get_available_audio_inputs() const override { + return { { 0, "Embedded" } }; + } + void set_audio_input(uint32_t audio_input_id) override {} // Ignore. + uint32_t get_current_audio_input() const override { return 0; } + +private: + void producer_thread_func(); + void send_disconnected_frame(); + bool play_video(const std::string &pathname); + void internal_rewind(); + + // Returns true if there was an error. + bool process_queued_commands(AVFormatContext *format_ctx, const std::string &pathname, timespec last_modified, bool *rewound); + + // Returns nullptr if no frame was decoded (e.g. EOF). + AVFrameWithDeleter decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx, + const std::string &pathname, int video_stream_index, int audio_stream_index, + bmusb::FrameAllocator::Frame *audio_frame, bmusb::AudioFormat *audio_format, int64_t *audio_pts, bool *error); + void convert_audio(const AVFrame *audio_avframe, bmusb::FrameAllocator::Frame *audio_frame, bmusb::AudioFormat *audio_format); + + bmusb::VideoFormat construct_video_format(const AVFrame *frame, AVRational video_timebase); + UniqueFrame make_video_frame(const AVFrame *frame, const std::string &pathname, bool *error); + + static int interrupt_cb_thunk(void *unique); + int interrupt_cb(); + + mutable std::mutex filename_mu; + std::string description, filename; + uint16_t timecode = 0; + unsigned width, height; + bmusb::PixelFormat pixel_format = bmusb::PixelFormat_8BitBGRA; + movit::YCbCrFormat current_frame_ycbcr_format; + bool running = false; + int card_index = -1; + double rate = 1.0; + std::atomic should_interrupt{false}; + bool last_frame_was_connected = true; + + bool has_dequeue_callbacks = false; + std::function dequeue_init_callback = nullptr; + std::function dequeue_cleanup_callback = nullptr; + + bmusb::FrameAllocator *video_frame_allocator = nullptr; + bmusb::FrameAllocator *audio_frame_allocator = nullptr; + std::unique_ptr owned_video_frame_allocator; + std::unique_ptr owned_audio_frame_allocator; + frame_callback_t frame_callback = nullptr; + audio_callback_t audio_callback = nullptr; + + SwsContextWithDeleter sws_ctx; + int sws_last_width = -1, sws_last_height = -1, sws_last_src_format = -1; + AVPixelFormat sws_dst_format = AVPixelFormat(-1); // In practice, always initialized. + AVRational video_timebase, audio_timebase; + + QuittableSleeper producer_thread_should_quit; + std::thread producer_thread; + + int64_t pts_origin, last_pts; + std::chrono::steady_clock::time_point start, next_frame_start, last_frame; + + std::mutex queue_mu; + struct QueuedCommand { + enum Command { REWIND, CHANGE_RATE } command; + double new_rate; // For CHANGE_RATE. + }; + std::vector command_queue; // Protected by . + + // Audio resampler. + AVAudioResampleContext *resampler = nullptr; + AVSampleFormat last_src_format, last_dst_format; + int64_t last_channel_layout; + int last_sample_rate; + +}; + +#endif // !defined(_FFMPEG_CAPTURE_H) diff --git a/nageru/ffmpeg_raii.cpp b/nageru/ffmpeg_raii.cpp new file mode 100644 index 0000000..746e03d --- /dev/null +++ b/nageru/ffmpeg_raii.cpp @@ -0,0 +1,77 @@ +#include "ffmpeg_raii.h" + +extern "C" { +#include +#include +#include +#include +#include +} + +using namespace std; + +// AVFormatContext + +void avformat_close_input_unique::operator() (AVFormatContext *format_ctx) const +{ + avformat_close_input(&format_ctx); +} + +AVFormatContextWithCloser avformat_open_input_unique( + const char *pathname, AVInputFormat *fmt, + AVDictionary **options) +{ + return avformat_open_input_unique(pathname, fmt, options, AVIOInterruptCB{ nullptr, nullptr }); +} + +AVFormatContextWithCloser avformat_open_input_unique( + const char *pathname, AVInputFormat *fmt, + AVDictionary **options, + const AVIOInterruptCB &interrupt_cb) +{ + AVFormatContext *format_ctx = avformat_alloc_context(); + format_ctx->interrupt_callback = interrupt_cb; + if (avformat_open_input(&format_ctx, pathname, fmt, options) != 0) { + format_ctx = nullptr; + } + return AVFormatContextWithCloser(format_ctx); +} + +// AVCodecContext + +void avcodec_free_context_unique::operator() (AVCodecContext *codec_ctx) const +{ + avcodec_free_context(&codec_ctx); +} + +AVCodecContextWithDeleter avcodec_alloc_context3_unique(const AVCodec *codec) +{ + return AVCodecContextWithDeleter(avcodec_alloc_context3(codec)); +} + + +// AVCodecParameters + +void avcodec_parameters_free_unique::operator() (AVCodecParameters *codec_par) const +{ + avcodec_parameters_free(&codec_par); +} + +// AVFrame + +void av_frame_free_unique::operator() (AVFrame *frame) const +{ + av_frame_free(&frame); +} + +AVFrameWithDeleter av_frame_alloc_unique() +{ + return AVFrameWithDeleter(av_frame_alloc()); +} + +// SwsContext + +void sws_free_context_unique::operator() (SwsContext *context) const +{ + sws_freeContext(context); +} diff --git a/nageru/ffmpeg_raii.h b/nageru/ffmpeg_raii.h new file mode 100644 index 0000000..33d2334 --- /dev/null +++ b/nageru/ffmpeg_raii.h @@ -0,0 +1,80 @@ +#ifndef _FFMPEG_RAII_H +#define _FFMPEG_RAII_H 1 + +// Some helpers to make RAII versions of FFmpeg objects. +// The cleanup functions don't interact all that well with unique_ptr, +// so things get a bit messy and verbose, but overall it's worth it to ensure +// we never leak things by accident in error paths. +// +// This does not cover any of the types that can actually be declared as +// a unique_ptr with no helper functions for deleter. + +#include + +struct AVCodec; +struct AVCodecContext; +struct AVCodecParameters; +struct AVDictionary; +struct AVFormatContext; +struct AVFrame; +struct AVInputFormat; +struct SwsContext; +typedef struct AVIOInterruptCB AVIOInterruptCB; + +// AVFormatContext +struct avformat_close_input_unique { + void operator() (AVFormatContext *format_ctx) const; +}; + +typedef std::unique_ptr + AVFormatContextWithCloser; + +AVFormatContextWithCloser avformat_open_input_unique( + const char *pathname, AVInputFormat *fmt, + AVDictionary **options); + +AVFormatContextWithCloser avformat_open_input_unique( + const char *pathname, AVInputFormat *fmt, + AVDictionary **options, + const AVIOInterruptCB &interrupt_cb); + + +// AVCodecContext +struct avcodec_free_context_unique { + void operator() (AVCodecContext *ctx) const; +}; + +typedef std::unique_ptr + AVCodecContextWithDeleter; + +AVCodecContextWithDeleter avcodec_alloc_context3_unique(const AVCodec *codec); + + +// AVCodecParameters +struct avcodec_parameters_free_unique { + void operator() (AVCodecParameters *codec_par) const; +}; + +typedef std::unique_ptr + AVCodecParametersWithDeleter; + + +// AVFrame +struct av_frame_free_unique { + void operator() (AVFrame *frame) const; +}; + +typedef std::unique_ptr + AVFrameWithDeleter; + +AVFrameWithDeleter av_frame_alloc_unique(); + +// SwsContext +struct sws_free_context_unique { + void operator() (SwsContext *context) const; +}; + +typedef std::unique_ptr + SwsContextWithDeleter; + +#endif // !defined(_FFMPEG_RAII_H) diff --git a/nageru/ffmpeg_util.cpp b/nageru/ffmpeg_util.cpp new file mode 100644 index 0000000..e348d0a --- /dev/null +++ b/nageru/ffmpeg_util.cpp @@ -0,0 +1,75 @@ +#include "ffmpeg_util.h" + +#include +#include +#include + +#include +#include + +#include "flags.h" + +using namespace std; + +string search_for_file(const string &filename) +{ + if (!filename.empty() && filename[0] == '/') { + // Absolute path. + return filename; + } + + // See if we match ^[a-z]:/, which is probably a URL of some sort + // (FFmpeg understands various forms of these). + for (size_t i = 0; i < filename.size() - 1; ++i) { + if (filename[i] == ':' && filename[i + 1] == '/') { + return filename; + } + if (!isalpha(filename[i])) { + break; + } + } + + // Look for the file in all theme_dirs until we find one; + // that will be the permanent resolution of this file, whether + // it is actually valid or not. + // We store errors from all the attempts, and show them + // once we know we can't find any of them. + vector errors; + for (const string &dir : global_flags.theme_dirs) { + string pathname = dir + "/" + filename; + if (access(pathname.c_str(), O_RDONLY) == 0) { + return pathname; + } else { + char buf[512]; + snprintf(buf, sizeof(buf), "%s: %s", pathname.c_str(), strerror(errno)); + errors.push_back(buf); + } + } + + for (const string &error : errors) { + fprintf(stderr, "%s\n", error.c_str()); + } + return ""; +} + +string search_for_file_or_die(const string &filename) +{ + string pathname = search_for_file(filename); + if (pathname.empty()) { + fprintf(stderr, "Couldn't find %s in any directory in --theme-dirs, exiting.\n", + filename.c_str()); + exit(1); + } + return pathname; +} + +int find_stream_index(AVFormatContext *ctx, AVMediaType media_type) +{ + for (unsigned i = 0; i < ctx->nb_streams; ++i) { + if (ctx->streams[i]->codecpar->codec_type == media_type) { + return i; + } + } + return -1; +} + diff --git a/nageru/ffmpeg_util.h b/nageru/ffmpeg_util.h new file mode 100644 index 0000000..c037a15 --- /dev/null +++ b/nageru/ffmpeg_util.h @@ -0,0 +1,23 @@ +#ifndef _FFMPEG_UTIL_H +#define _FFMPEG_UTIL_H 1 + +// Some common utilities for the two FFmpeg users (ImageInput and FFmpegCapture). + +#include + +extern "C" { +#include +} + +// Look for the file in all theme_dirs until we find one; +// that will be the permanent resolution of this file, whether +// it is actually valid or not. Returns an empty string on error. +std::string search_for_file(const std::string &filename); + +// Same, but exits on error. +std::string search_for_file_or_die(const std::string &filename); + +// Returns -1 if not found. +int find_stream_index(AVFormatContext *ctx, AVMediaType media_type); + +#endif // !defined(_FFMPEG_UTIL_H) diff --git a/nageru/filter.cpp b/nageru/filter.cpp new file mode 100644 index 0000000..0cb0180 --- /dev/null +++ b/nageru/filter.cpp @@ -0,0 +1,393 @@ +#include +#include +#include +#include +#include +#include + +#include "defs.h" + +#ifdef __SSE__ +#include +#endif + +#include "filter.h" + +using namespace std; + +#ifdef __SSE__ + +// For SSE, we set the denormals-as-zero flag instead. +#define early_undenormalise(sample) + +#else // !defined(__SSE__) + +union uint_float { + float f; + unsigned int i; +}; +#define early_undenormalise(sample) { \ + uint_float uf; \ + uf.f = float(sample); \ + if ((uf.i&0x60000000)==0) sample=0.0f; \ +} + +#endif // !_defined(__SSE__) + +Filter::Filter() +{ + omega = M_PI; + resonance = 0.01f; + A = 1.0f; + + init(FILTER_NONE, 1); + update(); +} + +void Filter::update() +{ + /* + uses coefficients grabbed from + RBJs audio eq cookbook: + http://www.musicdsp.org/files/Audio-EQ-Cookbook.txt + */ + + float sn, cs; + float cutoff_freq = omega; + cutoff_freq = min(cutoff_freq, (float)M_PI); + cutoff_freq = max(cutoff_freq, 0.001f); + calcSinCos(cutoff_freq, &sn, &cs); + if (resonance <= 0) resonance = 0.001f; + +#ifdef __GNUC__ + // Faster version of real_resonance = resonance ^ (1 / order). + // pow(), at least on current GCC, is pretty slow. + float real_resonance = resonance; + switch (filter_order) { + case 0: + case 1: + break; + case 4: + real_resonance = sqrt(real_resonance); + // Fall through. + case 2: + real_resonance = sqrt(real_resonance); + break; + case 3: + real_resonance = cbrt(real_resonance); + break; + default: + assert(false); + } +#else + float real_resonance = pow(resonance, 1.0f / filter_order); +#endif + + float alpha = float(sn / (2 * real_resonance)); + float a0 = 1 + alpha; + a1 = -2 * cs; + a2 = 1 - alpha; + + switch (filtertype) { + case FILTER_NONE: + a0 = b0 = 1.0f; + a1 = a2 = b1 = b2 = 0.0; //identity filter + break; + + case FILTER_LPF: + b0 = (1 - cs) * 0.5f; + b1 = 1 - cs; + b2 = b0; + // a1 = -2*cs; + // a2 = 1 - alpha; + break; + + case FILTER_HPF: + b0 = (1 + cs) * 0.5f; + b1 = -(1 + cs); + b2 = b0; + // a1 = -2*cs; + // a2 = 1 - alpha; + break; + + case FILTER_BPF: + b0 = alpha; + b1 = 0.0f; + b2 = -alpha; + // a1 = -2*cs; + // a2 = 1 - alpha; + break; + + case FILTER_NOTCH: + b0 = 1.0f; + b1 = -2*cs; + b2 = 1.0f; + // a1 = -2*cs; + // a2 = 1 - alpha; + break; + + case FILTER_APF: + b0 = 1 - alpha; + b1 = -2*cs; + b2 = 1.0f; + // a1 = -2*cs; + // a2 = 1 - alpha; + break; + + case FILTER_PEAKING_EQ: + b0 = 1 + alpha * A; + b1 = -2*cs; + b2 = 1 - alpha * A; + a0 = 1 + alpha / A; + // a1 = -2*cs; + a2 = 1 - alpha / A; + break; + + case FILTER_LOW_SHELF: + b0 = A * ((A + 1) - (A - 1)*cs + 2 * sqrt(A) * alpha); + b1 = 2 * A * ((A - 1) - (A + 1)*cs ); + b2 = A * ((A + 1) - (A - 1)*cs - 2 * sqrt(A) * alpha); + a0 = (A + 1) + (A - 1)*cs + 2 * sqrt(A) * alpha ; + a1 = -2 * ((A - 1) + (A + 1)*cs ); + a2 = (A + 1) + (A - 1)*cs - 2 * sqrt(A) * alpha ; + break; + + case FILTER_HIGH_SHELF: + b0 = A * ((A + 1) + (A - 1)*cs + 2 * sqrt(A) * alpha); + b1 = -2 * A * ((A - 1) + (A + 1)*cs ); + b2 = A * ((A + 1) + (A - 1)*cs - 2 * sqrt(A) * alpha); + a0 = (A + 1) - (A - 1)*cs + 2 * sqrt(A) * alpha ; + a1 = 2 * ((A - 1) - (A + 1)*cs ); + a2 = (A + 1) - (A - 1)*cs - 2 * sqrt(A) * alpha ; + break; + + default: + //unknown filter type + assert(false); + break; + } + + const float invA0 = 1.0f / a0; + b0 *= invA0; + b1 *= invA0; + b2 *= invA0; + a1 *= invA0; + a2 *= invA0; +} + +#ifndef NDEBUG +void Filter::debug() +{ + // Feed this to gnuplot to get a graph of the frequency response. + const float Fs2 = OUTPUT_FREQUENCY * 0.5f; + printf("set xrange [2:%f]; ", Fs2); + printf("set yrange [-80:20]; "); + printf("set log x; "); + printf("phasor(x) = cos(x*pi/%f)*{1,0} + sin(x*pi/%f)*{0,1}; ", Fs2, Fs2); + printf("tfunc(x, b0, b1, b2, a0, a1, a2) = (b0 * phasor(x)**2 + b1 * phasor(x) + b2) / (a0 * phasor(x)**2 + a1 * phasor(x) + a2); "); + printf("db(x) = 20*log10(x); "); + printf("plot db(abs(tfunc(x, %f, %f, %f, %f, %f, %f))) title \"\"\n", b0, b1, b2, 1.0f, a1, a2); +} +#endif + +void Filter::init(FilterType type, int order) +{ + filtertype = type; + filter_order = order; + if (filtertype == FILTER_NONE) filter_order = 0; + if (filter_order == 0) filtertype = FILTER_NONE; + + //reset feedback buffer + for (unsigned i = 0; i < filter_order; i++) { + feedback[i].d0 = feedback[i].d1 = 0.0f; + } +} + +#ifdef __SSE__ +void Filter::render_chunk(float *inout_buf, unsigned int n_samples) +#else +void Filter::render_chunk(float *inout_buf, unsigned int n_samples, unsigned stride) +#endif +{ +#ifdef __SSE__ + const unsigned stride = 1; + unsigned old_denormals_mode = _MM_GET_FLUSH_ZERO_MODE(); + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); +#endif + assert((n_samples & 3) == 0); // make sure n_samples is divisible by 4 + + // Apply the filter FILTER_ORDER times. + for (unsigned j = 0; j < filter_order; j++) { + float d0 = feedback[j].d0; + float d1 = feedback[j].d1; + float *inout_ptr = inout_buf; + + // Render n_samples mono samples. Unrolling manually by a + // factor four seemingly helps a lot, perhaps because it + // lets the CPU overlap arithmetic and memory operations + // better, or perhaps simply because the loop overhead is + // high. + for (unsigned i = n_samples >> 2; i; i--) { + float in, out; + + in = *inout_ptr; + out = b0*in + d0; + *inout_ptr = out; + d0 = b1*in - a1*out + d1; + d1 = b2*in - a2*out; + inout_ptr += stride; + + in = *inout_ptr; + out = b0*in + d0; + *inout_ptr = out; + d0 = b1*in - a1*out + d1; + d1 = b2*in - a2*out; + inout_ptr += stride; + + in = *inout_ptr; + out = b0*in + d0; + *inout_ptr = out; + d0 = b1*in - a1*out + d1; + d1 = b2*in - a2*out; + inout_ptr += stride; + + in = *inout_ptr; + out = b0*in + d0; + *inout_ptr = out; + d0 = b1*in - a1*out + d1; + d1 = b2*in - a2*out; + inout_ptr += stride; + } + early_undenormalise(d0); //do denormalization step + early_undenormalise(d1); + feedback[j].d0 = d0; + feedback[j].d1 = d1; + } + +#ifdef __SSE__ + _MM_SET_FLUSH_ZERO_MODE(old_denormals_mode); +#endif +} + +void Filter::render(float *inout_buf, unsigned int buf_size, float cutoff, float resonance) +{ + //render buf_size mono samples +#ifdef __SSE__ + assert(buf_size % 4 == 0); +#endif + if (filter_order == 0) + return; + + this->set_linear_cutoff(cutoff); + this->set_resonance(resonance); + this->update(); + this->render_chunk(inout_buf, buf_size); +} + +void StereoFilter::init(FilterType type, int new_order) +{ +#ifdef __SSE__ + parm_filter.init(type, new_order); + memset(feedback, 0, sizeof(feedback)); +#else + for (unsigned i = 0; i < 2; ++i) { + filters[i].init(type, new_order); + } +#endif +} + +void StereoFilter::render(float *inout_left_ptr, unsigned n_samples, float cutoff, float resonance, float dbgain_normalized) +{ +#ifdef __SSE__ + if (parm_filter.filtertype == FILTER_NONE || parm_filter.filter_order == 0) + return; + + unsigned old_denormals_mode = _MM_GET_FLUSH_ZERO_MODE(); + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + + parm_filter.set_linear_cutoff(cutoff); + parm_filter.set_resonance(resonance); + parm_filter.set_dbgain_normalized(dbgain_normalized); + parm_filter.update(); + + __m128 b0 = _mm_set1_ps(parm_filter.b0); + __m128 b1 = _mm_set1_ps(parm_filter.b1); + __m128 b2 = _mm_set1_ps(parm_filter.b2); + __m128 a1 = _mm_set1_ps(parm_filter.a1); + __m128 a2 = _mm_set1_ps(parm_filter.a2); + + // Apply the filter FILTER_ORDER times. + for (unsigned j = 0; j < parm_filter.filter_order; j++) { + __m128 d0 = feedback[j].d0; + __m128 d1 = feedback[j].d1; + __m64 *inout_ptr = (__m64 *)inout_left_ptr; + + __m128 in = _mm_set1_ps(0.0f), out; + for (unsigned i = n_samples; i; i--) { + in = _mm_loadl_pi(in, inout_ptr); + out = _mm_add_ps(_mm_mul_ps(b0, in), d0); + _mm_storel_pi(inout_ptr, out); + d0 = _mm_add_ps(_mm_sub_ps(_mm_mul_ps(b1, in), _mm_mul_ps(a1, out)), d1); + d1 = _mm_sub_ps(_mm_mul_ps(b2, in), _mm_mul_ps(a2, out)); + ++inout_ptr; + } + feedback[j].d0 = d0; + feedback[j].d1 = d1; + } + + _MM_SET_FLUSH_ZERO_MODE(old_denormals_mode); +#else + if (filters[0].filtertype == FILTER_NONE || filters[0].filter_order == 0) + return; + + for (unsigned i = 0; i < 2; ++i) { + filters[i].set_linear_cutoff(cutoff); + filters[i].set_resonance(resonance); + filters[i].update(); + filters[i].render_chunk(inout_left_ptr, n_samples, 2); + + ++inout_left_ptr; + } +#endif +} + +/* + + Find the transfer function for an IIR biquad. This is relatively basic signal + processing, but for completeness, here's the rationale for the function: + + The basic system of an IIR biquad looks like this, for input x[n], output y[n] + and constant filter coefficients [ab][0-2]: + + a2 y[n-2] + a1 y[n-1] + a0 y[n] = b2 x[n-2] + b1 x[n-1] + b0 x[n] + + Taking the discrete Fourier transform (DFT) of both sides (denoting by convention + DFT{x[n]} by X[w], where w is the angular frequency, going from 0 to 2pi), yields, + due to the linearity and shift properties of the DFT: + + a2 e^2jw Y[w] + a1 e^jw Y[w] + a0 Y[w] = b2 e^2jw X[w] + b1 e^jw X[w] + b0 Y[w] + + Simple factorization and reorganization yields + + Y[w] / X[w] = (b1 e^2jw + b1 e^jw + b0) / (a2 e^2jw + a1 e^jw + a0) + + and Y[w] / X[w] is by definition the filter's _transfer function_ + (customarily denoted by H(w)), ie. the complex factor it applies to the + frequency component w. The absolute value of the transfer function is + the frequency response, ie. how much frequency w is boosted or weakened. + + (This derivation usually goes via the Z-transform and not the DFT, but the + idea is exactly the same; the Z-transform is just a bit more general.) + + Sending a signal through first one filter and then through another one + will naturally be equivalent to a filter with the transfer function equal + to the pointwise multiplication of the two filters, so for N-order filters + we need to raise the answer to the Nth power. + +*/ +complex Filter::evaluate_transfer_function(float omega) +{ + complex z = exp(complex(0.0f, omega)); + complex z2 = z * z; + return pow((b0 * z2 + b1 * z + b2) / (1.0f * z2 + a1 * z + a2), filter_order); +} diff --git a/nageru/filter.h b/nageru/filter.h new file mode 100644 index 0000000..1bf18c9 --- /dev/null +++ b/nageru/filter.h @@ -0,0 +1,139 @@ +// Filter class: +// a cascaded biquad IIR filter +// +// Special cases for type=LPF/BPF/HPF: +// +// Butterworth filter: order=1, resonance=1/sqrt(2) +// Linkwitz-Riley filter: order=2, resonance=1/2 + +#ifndef _FILTER_H +#define _FILTER_H 1 + +#define _USE_MATH_DEFINES +#include +#include + +#ifdef __SSE__ +#include +#endif + +enum FilterType +{ + FILTER_NONE = 0, + FILTER_LPF, + FILTER_HPF, + FILTER_BPF, + FILTER_NOTCH, + FILTER_APF, + + // EQ filters. + FILTER_PEAKING_EQ, + FILTER_LOW_SHELF, + FILTER_HIGH_SHELF, +}; + +#define FILTER_MAX_ORDER 4 + +class Filter +{ + friend class StereoFilter; + friend class SplittingStereoFilter; +public: + Filter(); + + void init(FilterType type, int new_order); + + void update(); //update coefficients +#ifndef NDEBUG + void debug(); +#endif + std::complex evaluate_transfer_function(float omega); + + FilterType get_type() { return filtertype; } + unsigned get_order() { return filter_order; } + + // cutoff is taken to be in the [0..pi> (see set_linear_cutoff, below). + void render(float *inout_array, unsigned int buf_size, float cutoff, float resonance); + + // Set cutoff, from [0..pi> (where pi is the Nyquist frequency). + // Overridden by render() if you use that. + void set_linear_cutoff(float new_omega) + { + omega = new_omega; + } + + void set_resonance(float new_resonance) + { + resonance = new_resonance; + } + + // For EQ filters only. + void set_dbgain_normalized(float db_gain_div_40) + { + A = pow(10.0f, db_gain_div_40); + } + +#ifdef __SSE__ + // We don't need the stride argument for SSE, as StereoFilter + // has its own SSE implementations. + void render_chunk(float *inout_buf, unsigned nSamples); +#else + void render_chunk(float *inout_buf, unsigned nSamples, unsigned stride = 1); +#endif + + FilterType filtertype; +private: + float omega; //which is 2*Pi*frequency /SAMPLE_RATE + float resonance; + float A; // which is 10^(db_gain / 40) + +public: + unsigned filter_order; +private: + float b0, b1, b2, a1, a2; //filter coefs + + struct FeedbackBuffer { + float d0,d1; //feedback buffers + } feedback[FILTER_MAX_ORDER]; + + void calcSinCos(float omega, float *sinVal, float *cosVal) + { + *sinVal = (float)sin(omega); + *cosVal = (float)cos(omega); + } +}; + + +class StereoFilter +{ +public: + void init(FilterType type, int new_order); + + void render(float *inout_left_ptr, unsigned n_samples, float cutoff, float resonance, float dbgain_normalized = 0.0f); +#ifndef NDEBUG +#ifdef __SSE__ + void debug() { parm_filter.debug(); } +#else + void debug() { filters[0].debug(); } +#endif +#endif +#ifdef __SSE__ + FilterType get_type() { return parm_filter.get_type(); } +#else + FilterType get_type() { return filters[0].get_type(); } +#endif + +private: +#ifdef __SSE__ + // We only use the filter to calculate coefficients; we don't actually + // use its feedbacks. + Filter parm_filter; + struct SIMDFeedbackBuffer { + __m128 d0, d1; + } feedback[FILTER_MAX_ORDER]; +#else + Filter filters[2]; +#endif +}; + +#endif // !defined(_FILTER_H) diff --git a/nageru/flags.cpp b/nageru/flags.cpp new file mode 100644 index 0000000..9b3a9da --- /dev/null +++ b/nageru/flags.cpp @@ -0,0 +1,604 @@ +#include "flags.h" + +#include +#include +#include +#include + +#include + +using namespace std; + +Flags global_flags; + +// Long options that have no corresponding short option. +enum LongOption { + OPTION_HELP = 1000, + OPTION_FULLSCREEN, + OPTION_MULTICHANNEL, + OPTION_MIDI_MAPPING, + OPTION_DEFAULT_HDMI_INPUT, + OPTION_FAKE_CARDS_AUDIO, + OPTION_HTTP_UNCOMPRESSED_VIDEO, + OPTION_HTTP_X264_VIDEO, + OPTION_RECORD_X264_VIDEO, + OPTION_X264_PRESET, + OPTION_X264_TUNE, + OPTION_X264_SPEEDCONTROL, + OPTION_X264_SPEEDCONTROL_VERBOSE, + OPTION_X264_BITRATE, + OPTION_X264_CRF, + OPTION_X264_VBV_BUFSIZE, + OPTION_X264_VBV_MAX_BITRATE, + OPTION_X264_PARAM, + OPTION_HTTP_MUX, + OPTION_HTTP_COARSE_TIMEBASE, + OPTION_HTTP_AUDIO_CODEC, + OPTION_HTTP_AUDIO_BITRATE, + OPTION_HTTP_PORT, + OPTION_NO_TRANSCODE_AUDIO, + OPTION_FLAT_AUDIO, + OPTION_GAIN_STAGING, + OPTION_DISABLE_LOCUT, + OPTION_ENABLE_LOCUT, + OPTION_DISABLE_GAIN_STAGING_AUTO, + OPTION_ENABLE_GAIN_STAGING_AUTO, + OPTION_DISABLE_COMPRESSOR, + OPTION_ENABLE_COMPRESSOR, + OPTION_DISABLE_LIMITER, + OPTION_ENABLE_LIMITER, + OPTION_DISABLE_MAKEUP_GAIN_AUTO, + OPTION_ENABLE_MAKEUP_GAIN_AUTO, + OPTION_DISABLE_ALSA_OUTPUT, + OPTION_NO_FLUSH_PBOS, + OPTION_PRINT_VIDEO_LATENCY, + OPTION_MAX_INPUT_QUEUE_FRAMES, + OPTION_AUDIO_QUEUE_LENGTH_MS, + OPTION_OUTPUT_YCBCR_COEFFICIENTS, + OPTION_OUTPUT_BUFFER_FRAMES, + OPTION_OUTPUT_SLOP_FRAMES, + OPTION_TIMECODE_STREAM, + OPTION_TIMECODE_STDOUT, + OPTION_QUICK_CUT_KEYS, + OPTION_10_BIT_INPUT, + OPTION_10_BIT_OUTPUT, + OPTION_INPUT_YCBCR_INTERPRETATION, +}; + +void usage(Program program) +{ + if (program == PROGRAM_KAERU) { + fprintf(stderr, "Usage: kaeru [OPTION]... SOURCE_URL\n"); + } else { + fprintf(stderr, "Usage: nageru [OPTION]...\n"); + } + fprintf(stderr, "\n"); + fprintf(stderr, " --help print usage information\n"); + if (program == PROGRAM_NAGERU) { + fprintf(stderr, " --fullscreen run in full screen, with no decorations\n"); + } + fprintf(stderr, " -w, --width output width in pixels (default 1280)\n"); + fprintf(stderr, " -h, --height output height in pixels (default 720)\n"); + if (program == PROGRAM_NAGERU) { + fprintf(stderr, " -c, --num-cards set number of input cards (default 2)\n"); + fprintf(stderr, " -o, --output-card=CARD also output signal to the given card (default none)\n"); + fprintf(stderr, " -t, --theme=FILE choose theme (default theme.lua)\n"); + fprintf(stderr, " -I, --theme-dir=DIR search for theme in this directory (can be given multiple times)\n"); + fprintf(stderr, " -r, --recording-dir=DIR where to store disk recording\n"); + fprintf(stderr, " -v, --va-display=SPEC VA-API device for H.264 encoding\n"); + fprintf(stderr, " ($DISPLAY spec or /dev/dri/render* path)\n"); + fprintf(stderr, " -m, --map-signal=SIGNAL,CARD set a default card mapping (can be given multiple times)\n"); + fprintf(stderr, " -M, --input-mapping=FILE start with the given audio input mapping (implies --multichannel)\n"); + fprintf(stderr, " --multichannel start in multichannel audio mapping mode\n"); + fprintf(stderr, " --midi-mapping=FILE start with the given MIDI controller mapping (implies --multichannel)\n"); + fprintf(stderr, " --default-hdmi-input default to HDMI over SDI inputs for cards that have both\n"); + fprintf(stderr, " --fake-cards-audio make fake (disconnected) cards output a simple tone\n"); + fprintf(stderr, " --http-uncompressed-video send uncompressed NV12 video to HTTP clients\n"); + fprintf(stderr, " --http-x264-video send x264-compressed video to HTTP clients\n"); + fprintf(stderr, " --record-x264-video store x264-compressed video to disk (implies --http-x264-video,\n"); + fprintf(stderr, " removes the need for working VA-API encoding)\n"); + } + fprintf(stderr, " --x264-preset x264 quality preset (default " X264_DEFAULT_PRESET ")\n"); + fprintf(stderr, " --x264-tune x264 tuning (default " X264_DEFAULT_TUNE ", can be blank)\n"); + fprintf(stderr, " --x264-speedcontrol try to match x264 preset to available CPU speed\n"); + fprintf(stderr, " --x264-speedcontrol-verbose output speedcontrol debugging statistics\n"); + fprintf(stderr, " --x264-bitrate x264 bitrate (in kilobit/sec, default %d)\n", + DEFAULT_X264_OUTPUT_BIT_RATE); + fprintf(stderr, " --x264-crf=VALUE quality-based VBR (-12 to 51), incompatible with --x264-bitrate and VBV\n"); + fprintf(stderr, " --x264-vbv-bufsize x264 VBV size (in kilobits, 0 = one-frame VBV,\n"); + fprintf(stderr, " default: same as --x264-bitrate, that is, one-second VBV)\n"); + fprintf(stderr, " --x264-vbv-max-bitrate x264 local max bitrate (in kilobit/sec per --vbv-bufsize,\n"); + fprintf(stderr, " 0 = no limit, default: same as --x264-bitrate, i.e., CBR)\n"); + fprintf(stderr, " --x264-param=NAME[,VALUE] set any x264 parameter, for fine tuning\n"); + fprintf(stderr, " --http-mux=NAME mux to use for HTTP streams (default " DEFAULT_STREAM_MUX_NAME ")\n"); + fprintf(stderr, " --http-audio-codec=NAME audio codec to use for HTTP streams\n"); + fprintf(stderr, " (default is to use the same as for the recording)\n"); + fprintf(stderr, " --http-audio-bitrate=KBITS audio codec bit rate to use for HTTP streams\n"); + fprintf(stderr, " (default is %d, ignored unless --http-audio-codec is set)\n", + DEFAULT_AUDIO_OUTPUT_BIT_RATE / 1000); + fprintf(stderr, " --http-port=PORT which port to use for the built-in HTTP server\n"); + fprintf(stderr, " (default is %d)\n", DEFAULT_HTTPD_PORT); + if (program == PROGRAM_KAERU) { + fprintf(stderr, " --no-transcode-audio copy encoded audio raw from the source stream\n"); + fprintf(stderr, " (requires --http-audio-codec= to be set)\n"); + } + if (program == PROGRAM_NAGERU) { + fprintf(stderr, " --flat-audio start with most audio processing turned off\n"); + fprintf(stderr, " (can be overridden by e.g. --enable-limiter)\n"); + fprintf(stderr, " --gain-staging=DB set initial gain staging to the given value\n"); + fprintf(stderr, " (--disable-gain-staging-auto)\n"); + fprintf(stderr, " --disable-locut turn off locut filter (also --enable)\n"); + fprintf(stderr, " --disable-gain-staging-auto turn off automatic gain staging (also --enable)\n"); + fprintf(stderr, " --disable-compressor turn off regular compressor (also --enable)\n"); + fprintf(stderr, " --disable-limiter turn off limiter (also --enable)\n"); + fprintf(stderr, " --disable-makeup-gain-auto turn off auto-adjustment of final makeup gain (also --enable)\n"); + fprintf(stderr, " --disable-alsa-output disable audio monitoring via ALSA\n"); + fprintf(stderr, " --no-flush-pbos do not explicitly signal texture data uploads\n"); + fprintf(stderr, " (will give display corruption, but makes it\n"); + fprintf(stderr, " possible to run with apitrace in real time)\n"); + fprintf(stderr, " --print-video-latency print out measurements of video latency on stdout\n"); + fprintf(stderr, " --max-input-queue-frames=FRAMES never keep more than FRAMES frames for each card\n"); + fprintf(stderr, " (default 6, minimum 1)\n"); + fprintf(stderr, " --audio-queue-length-ms=MS length of audio resampling queue (default 100.0)\n"); + fprintf(stderr, " --output-ycbcr-coefficients={rec601,rec709,auto}\n"); + fprintf(stderr, " Y'CbCr coefficient standard of output (default auto)\n"); + fprintf(stderr, " auto is rec601, unless --output-card is used\n"); + fprintf(stderr, " and a Rec. 709 mode (typically HD modes) is in use\n"); + fprintf(stderr, " --output-buffer-frames=NUM number of frames in output buffer for --output-card,\n"); + fprintf(stderr, " can be fractional (default 6.0); note also\n"); + fprintf(stderr, " the audio queue can't be much longer than this\n"); + fprintf(stderr, " --output-slop-frames=NUM if more less than this number of frames behind for\n"); + fprintf(stderr, " --output-card, try to submit anyway instead of\n"); + fprintf(stderr, " dropping the frame (default 0.5)\n"); + fprintf(stderr, " --timecode-stream show timestamp and timecode in stream\n"); + fprintf(stderr, " --timecode-stdout show timestamp and timecode on standard output\n"); + fprintf(stderr, " --quick-cut-keys enable direct cutting by Q, W, E, ... keys\n"); + fprintf(stderr, " --10-bit-input use 10-bit video input (requires compute shaders)\n"); + fprintf(stderr, " --10-bit-output use 10-bit video output (requires compute shaders,\n"); + fprintf(stderr, " implies --record-x264-video)\n"); + fprintf(stderr, " --input-ycbcr-interpretation=CARD,{rec601,rec709,auto}[,{limited,full}]\n"); + fprintf(stderr, " Y'CbCr coefficient standard of card CARD (default auto)\n"); + fprintf(stderr, " auto is rec601 for SD, rec709 for HD, always limited\n"); + fprintf(stderr, " limited means standard 0-240/0-235 input range (for 8-bit)\n"); + } +} + +void parse_flags(Program program, int argc, char * const argv[]) +{ + static const option long_options[] = { + { "help", no_argument, 0, OPTION_HELP }, + { "fullscreen", no_argument, 0, OPTION_FULLSCREEN }, + { "width", required_argument, 0, 'w' }, + { "height", required_argument, 0, 'h' }, + { "num-cards", required_argument, 0, 'c' }, + { "output-card", required_argument, 0, 'o' }, + { "theme", required_argument, 0, 't' }, + { "theme-dir", required_argument, 0, 'I' }, + { "recording-dir", required_argument, 0, 'r' }, + { "map-signal", required_argument, 0, 'm' }, + { "input-mapping", required_argument, 0, 'M' }, + { "va-display", required_argument, 0, 'v' }, + { "multichannel", no_argument, 0, OPTION_MULTICHANNEL }, + { "midi-mapping", required_argument, 0, OPTION_MIDI_MAPPING }, + { "default-hdmi-input", no_argument, 0, OPTION_DEFAULT_HDMI_INPUT }, + { "fake-cards-audio", no_argument, 0, OPTION_FAKE_CARDS_AUDIO }, + { "http-uncompressed-video", no_argument, 0, OPTION_HTTP_UNCOMPRESSED_VIDEO }, + { "http-x264-video", no_argument, 0, OPTION_HTTP_X264_VIDEO }, + { "record-x264-video", no_argument, 0, OPTION_RECORD_X264_VIDEO }, + { "x264-preset", required_argument, 0, OPTION_X264_PRESET }, + { "x264-tune", required_argument, 0, OPTION_X264_TUNE }, + { "x264-speedcontrol", no_argument, 0, OPTION_X264_SPEEDCONTROL }, + { "x264-speedcontrol-verbose", no_argument, 0, OPTION_X264_SPEEDCONTROL_VERBOSE }, + { "x264-bitrate", required_argument, 0, OPTION_X264_BITRATE }, + { "x264-crf", required_argument, 0, OPTION_X264_CRF }, + { "x264-vbv-bufsize", required_argument, 0, OPTION_X264_VBV_BUFSIZE }, + { "x264-vbv-max-bitrate", required_argument, 0, OPTION_X264_VBV_MAX_BITRATE }, + { "x264-param", required_argument, 0, OPTION_X264_PARAM }, + { "http-mux", required_argument, 0, OPTION_HTTP_MUX }, + { "http-audio-codec", required_argument, 0, OPTION_HTTP_AUDIO_CODEC }, + { "http-audio-bitrate", required_argument, 0, OPTION_HTTP_AUDIO_BITRATE }, + { "http-port", required_argument, 0, OPTION_HTTP_PORT }, + { "no-transcode-audio", no_argument, 0, OPTION_NO_TRANSCODE_AUDIO }, + { "flat-audio", no_argument, 0, OPTION_FLAT_AUDIO }, + { "gain-staging", required_argument, 0, OPTION_GAIN_STAGING }, + { "disable-locut", no_argument, 0, OPTION_DISABLE_LOCUT }, + { "enable-locut", no_argument, 0, OPTION_ENABLE_LOCUT }, + { "disable-gain-staging-auto", no_argument, 0, OPTION_DISABLE_GAIN_STAGING_AUTO }, + { "enable-gain-staging-auto", no_argument, 0, OPTION_ENABLE_GAIN_STAGING_AUTO }, + { "disable-compressor", no_argument, 0, OPTION_DISABLE_COMPRESSOR }, + { "enable-compressor", no_argument, 0, OPTION_ENABLE_COMPRESSOR }, + { "disable-limiter", no_argument, 0, OPTION_DISABLE_LIMITER }, + { "enable-limiter", no_argument, 0, OPTION_ENABLE_LIMITER }, + { "disable-makeup-gain-auto", no_argument, 0, OPTION_DISABLE_MAKEUP_GAIN_AUTO }, + { "enable-makeup-gain-auto", no_argument, 0, OPTION_ENABLE_MAKEUP_GAIN_AUTO }, + { "disable-alsa-output", no_argument, 0, OPTION_DISABLE_ALSA_OUTPUT }, + { "no-flush-pbos", no_argument, 0, OPTION_NO_FLUSH_PBOS }, + { "print-video-latency", no_argument, 0, OPTION_PRINT_VIDEO_LATENCY }, + { "max-input-queue-frames", required_argument, 0, OPTION_MAX_INPUT_QUEUE_FRAMES }, + { "audio-queue-length-ms", required_argument, 0, OPTION_AUDIO_QUEUE_LENGTH_MS }, + { "output-ycbcr-coefficients", required_argument, 0, OPTION_OUTPUT_YCBCR_COEFFICIENTS }, + { "output-buffer-frames", required_argument, 0, OPTION_OUTPUT_BUFFER_FRAMES }, + { "output-slop-frames", required_argument, 0, OPTION_OUTPUT_SLOP_FRAMES }, + { "timecode-stream", no_argument, 0, OPTION_TIMECODE_STREAM }, + { "timecode-stdout", no_argument, 0, OPTION_TIMECODE_STDOUT }, + { "quick-cut-keys", no_argument, 0, OPTION_QUICK_CUT_KEYS }, + { "10-bit-input", no_argument, 0, OPTION_10_BIT_INPUT }, + { "10-bit-output", no_argument, 0, OPTION_10_BIT_OUTPUT }, + { "input-ycbcr-interpretation", required_argument, 0, OPTION_INPUT_YCBCR_INTERPRETATION }, + { 0, 0, 0, 0 } + }; + vector theme_dirs; + string output_ycbcr_coefficients = "auto"; + for ( ;; ) { + int option_index = 0; + int c = getopt_long(argc, argv, "c:t:I:r:v:m:M:w:h:", long_options, &option_index); + + if (c == -1) { + break; + } + switch (c) { + case 'w': + global_flags.width = atoi(optarg); + break; + case 'h': + global_flags.height = atoi(optarg); + break; + case 'c': + global_flags.num_cards = atoi(optarg); + break; + case 'o': + global_flags.output_card = atoi(optarg); + break; + case 't': + global_flags.theme_filename = optarg; + break; + case 'I': + theme_dirs.push_back(optarg); + break; + case 'r': + global_flags.recording_dir = optarg; + break; + case 'm': { + char *ptr = strchr(optarg, ','); + if (ptr == nullptr) { + fprintf(stderr, "ERROR: Invalid argument '%s' to --map-signal (needs a signal and a card number, separated by comma)\n", optarg); + exit(1); + } + *ptr = '\0'; + const int signal_num = atoi(optarg); + const int card_num = atoi(ptr + 1); + if (global_flags.default_stream_mapping.count(signal_num)) { + fprintf(stderr, "ERROR: Signal %d already mapped to card %d\n", + signal_num, global_flags.default_stream_mapping[signal_num]); + exit(1); + } + global_flags.default_stream_mapping[signal_num] = card_num; + break; + } + case 'M': + global_flags.input_mapping_filename = optarg; + break; + case OPTION_MULTICHANNEL: + global_flags.multichannel_mapping_mode = true; + break; + case 'v': + global_flags.va_display = optarg; + break; + case OPTION_MIDI_MAPPING: + global_flags.midi_mapping_filename = optarg; + global_flags.multichannel_mapping_mode = true; + break; + case OPTION_DEFAULT_HDMI_INPUT: + global_flags.default_hdmi_input = true; + break; + case OPTION_FAKE_CARDS_AUDIO: + global_flags.fake_cards_audio = true; + break; + case OPTION_HTTP_UNCOMPRESSED_VIDEO: + global_flags.uncompressed_video_to_http = true; + break; + case OPTION_HTTP_MUX: + global_flags.stream_mux_name = optarg; + break; + case OPTION_HTTP_AUDIO_CODEC: + global_flags.stream_audio_codec_name = optarg; + break; + case OPTION_HTTP_AUDIO_BITRATE: + global_flags.stream_audio_codec_bitrate = atoi(optarg) * 1000; + break; + case OPTION_HTTP_PORT: + global_flags.http_port = atoi(optarg); + break; + case OPTION_NO_TRANSCODE_AUDIO: + global_flags.transcode_audio = false; + break; + case OPTION_HTTP_X264_VIDEO: + global_flags.x264_video_to_http = true; + break; + case OPTION_RECORD_X264_VIDEO: + global_flags.x264_video_to_disk = true; + global_flags.x264_video_to_http = true; + break; + case OPTION_X264_PRESET: + global_flags.x264_preset = optarg; + break; + case OPTION_X264_TUNE: + global_flags.x264_tune = optarg; + break; + case OPTION_X264_SPEEDCONTROL: + global_flags.x264_speedcontrol = true; + break; + case OPTION_X264_SPEEDCONTROL_VERBOSE: + global_flags.x264_speedcontrol_verbose = true; + break; + case OPTION_X264_BITRATE: + global_flags.x264_bitrate = atoi(optarg); + break; + case OPTION_X264_CRF: + global_flags.x264_crf = atof(optarg); + break; + case OPTION_X264_VBV_BUFSIZE: + global_flags.x264_vbv_buffer_size = atoi(optarg); + break; + case OPTION_X264_VBV_MAX_BITRATE: + global_flags.x264_vbv_max_bitrate = atoi(optarg); + break; + case OPTION_X264_PARAM: + global_flags.x264_extra_param.push_back(optarg); + break; + case OPTION_FLAT_AUDIO: + // If --flat-audio is given, turn off everything that messes with the sound, + // except the final makeup gain. + global_flags.locut_enabled = false; + global_flags.gain_staging_auto = false; + global_flags.compressor_enabled = false; + global_flags.limiter_enabled = false; + break; + case OPTION_GAIN_STAGING: + global_flags.initial_gain_staging_db = atof(optarg); + global_flags.gain_staging_auto = false; + break; + case OPTION_DISABLE_LOCUT: + global_flags.locut_enabled = false; + break; + case OPTION_ENABLE_LOCUT: + global_flags.locut_enabled = true; + break; + case OPTION_DISABLE_GAIN_STAGING_AUTO: + global_flags.gain_staging_auto = false; + break; + case OPTION_ENABLE_GAIN_STAGING_AUTO: + global_flags.gain_staging_auto = true; + break; + case OPTION_DISABLE_COMPRESSOR: + global_flags.compressor_enabled = false; + break; + case OPTION_ENABLE_COMPRESSOR: + global_flags.compressor_enabled = true; + break; + case OPTION_DISABLE_LIMITER: + global_flags.limiter_enabled = false; + break; + case OPTION_ENABLE_LIMITER: + global_flags.limiter_enabled = true; + break; + case OPTION_DISABLE_MAKEUP_GAIN_AUTO: + global_flags.final_makeup_gain_auto = false; + break; + case OPTION_ENABLE_MAKEUP_GAIN_AUTO: + global_flags.final_makeup_gain_auto = true; + break; + case OPTION_DISABLE_ALSA_OUTPUT: + global_flags.enable_alsa_output = false; + break; + case OPTION_NO_FLUSH_PBOS: + global_flags.flush_pbos = false; + break; + case OPTION_PRINT_VIDEO_LATENCY: + global_flags.print_video_latency = true; + break; + case OPTION_MAX_INPUT_QUEUE_FRAMES: + global_flags.max_input_queue_frames = atoi(optarg); + break; + case OPTION_AUDIO_QUEUE_LENGTH_MS: + global_flags.audio_queue_length_ms = atof(optarg); + break; + case OPTION_OUTPUT_YCBCR_COEFFICIENTS: + output_ycbcr_coefficients = optarg; + break; + case OPTION_OUTPUT_BUFFER_FRAMES: + global_flags.output_buffer_frames = atof(optarg); + break; + case OPTION_OUTPUT_SLOP_FRAMES: + global_flags.output_slop_frames = atof(optarg); + break; + case OPTION_TIMECODE_STREAM: + global_flags.display_timecode_in_stream = true; + break; + case OPTION_TIMECODE_STDOUT: + global_flags.display_timecode_on_stdout = true; + break; + case OPTION_QUICK_CUT_KEYS: + global_flags.enable_quick_cut_keys = true; + break; + case OPTION_10_BIT_INPUT: + global_flags.ten_bit_input = true; + break; + case OPTION_10_BIT_OUTPUT: + global_flags.ten_bit_output = true; + global_flags.x264_video_to_disk = true; + global_flags.x264_video_to_http = true; + global_flags.x264_bit_depth = 10; + break; + case OPTION_INPUT_YCBCR_INTERPRETATION: { + char *ptr = strchr(optarg, ','); + if (ptr == nullptr) { + fprintf(stderr, "ERROR: Invalid argument '%s' to --input-ycbcr-interpretation (needs a card and an interpretation, separated by comma)\n", optarg); + exit(1); + } + *ptr = '\0'; + const int card_num = atoi(optarg); + if (card_num < 0 || card_num >= MAX_VIDEO_CARDS) { + fprintf(stderr, "ERROR: Invalid card number %d\n", card_num); + exit(1); + } + + YCbCrInterpretation interpretation; + char *interpretation_str = ptr + 1; + ptr = strchr(interpretation_str, ','); + if (ptr != nullptr) { + *ptr = '\0'; + const char *range = ptr + 1; + if (strcmp(range, "full") == 0) { + interpretation.full_range = true; + } else if (strcmp(range, "limited") == 0) { + interpretation.full_range = false; + } else { + fprintf(stderr, "ERROR: Invalid Y'CbCr range '%s' (must be “full” or “limited”)\n", range); + exit(1); + } + } + + if (strcmp(interpretation_str, "rec601") == 0) { + interpretation.ycbcr_coefficients_auto = false; + interpretation.ycbcr_coefficients = movit::YCBCR_REC_601; + } else if (strcmp(interpretation_str, "rec709") == 0) { + interpretation.ycbcr_coefficients_auto = false; + interpretation.ycbcr_coefficients = movit::YCBCR_REC_709; + } else if (strcmp(interpretation_str, "auto") == 0) { + interpretation.ycbcr_coefficients_auto = true; + if (interpretation.full_range) { + fprintf(stderr, "ERROR: Cannot use “auto” Y'CbCr coefficients with full range\n"); + exit(1); + } + } else { + fprintf(stderr, "ERROR: Invalid Y'CbCr coefficients '%s' (must be “rec601”, “rec709” or “auto”)\n", interpretation_str); + exit(1); + } + global_flags.ycbcr_interpretation[card_num] = interpretation; + break; + } + case OPTION_FULLSCREEN: + global_flags.fullscreen = true; + break; + case OPTION_HELP: + usage(program); + exit(0); + default: + fprintf(stderr, "Unknown option '%s'\n", argv[option_index]); + fprintf(stderr, "\n"); + usage(program); + exit(1); + } + } + + if (global_flags.uncompressed_video_to_http && + global_flags.x264_video_to_http) { + fprintf(stderr, "ERROR: --http-uncompressed-video and --http-x264-video are mutually incompatible\n"); + exit(1); + } + if (global_flags.num_cards <= 0) { + fprintf(stderr, "ERROR: --num-cards must be at least 1\n"); + exit(1); + } + if (global_flags.output_card < -1 || + global_flags.output_card >= global_flags.num_cards) { + fprintf(stderr, "ERROR: --output-card points to a nonexistant card\n"); + exit(1); + } + if (!global_flags.transcode_audio && global_flags.stream_audio_codec_name.empty()) { + fprintf(stderr, "ERROR: If not transcoding audio, you must specify ahead-of-time what audio codec is in use\n"); + fprintf(stderr, " (using --http-audio-codec).\n"); + exit(1); + } + if (global_flags.x264_speedcontrol) { + if (!global_flags.x264_preset.empty() && global_flags.x264_preset != "faster") { + fprintf(stderr, "WARNING: --x264-preset is overridden by --x264-speedcontrol (implicitly uses \"faster\" as base preset)\n"); + } + global_flags.x264_preset = "faster"; + } else if (global_flags.x264_preset.empty()) { + global_flags.x264_preset = X264_DEFAULT_PRESET; + } + if (!theme_dirs.empty()) { + global_flags.theme_dirs = theme_dirs; + } + + // In reality, we could probably do with any even value (we subsample + // by two in some places), but it's better to be on the safe side + // wrt. video codecs and such. (I'd set 16 if I could, but 1080 isn't + // divisible by 16.) + if (global_flags.width <= 0 || (global_flags.width % 8) != 0 || + global_flags.height <= 0 || (global_flags.height % 8) != 0) { + fprintf(stderr, "ERROR: --width and --height must be positive integers divisible by 8\n"); + exit(1); + } + + for (pair mapping : global_flags.default_stream_mapping) { + if (mapping.second >= global_flags.num_cards) { + fprintf(stderr, "ERROR: Signal %d mapped to card %d, which doesn't exist (try adjusting --num-cards)\n", + mapping.first, mapping.second); + exit(1); + } + } + + // Rec. 709 would be the sane thing to do, but it seems many players + // just default to BT.601 coefficients no matter what. We _do_ set + // the right flags, so that a player that works properly doesn't have + // to guess, but it's frequently ignored. See discussions + // in e.g. https://trac.ffmpeg.org/ticket/4978; the situation with + // browsers is complicated and depends on things like hardware acceleration + // (https://bugs.chromium.org/p/chromium/issues/detail?id=333619 for + // extensive discussion). VLC generally fixed this as part of 3.0.0 + // (see e.g. https://github.com/videolan/vlc/commit/bc71288b2e38c07d6921472824b92eef1aa85f7e + // and https://github.com/videolan/vlc/commit/c3fc2683a9cde1d42674ebf9935dced05733a215), + // but earlier versions were pretty random. + // + // On the other hand, HDMI/SDI output typically requires Rec. 709 for + // HD resolutions (with no way of signaling anything else), which is + // a conflicting demand. In this case, we typically let the HDMI/SDI + // output win if it is active, but the user can override this. + if (output_ycbcr_coefficients == "auto") { + // Essentially: BT.709 if HDMI/SDI output is on, otherwise BT.601. + global_flags.ycbcr_rec709_coefficients = false; + global_flags.ycbcr_auto_coefficients = true; + } else if (output_ycbcr_coefficients == "rec709") { + global_flags.ycbcr_rec709_coefficients = true; + global_flags.ycbcr_auto_coefficients = false; + } else if (output_ycbcr_coefficients == "rec601") { + global_flags.ycbcr_rec709_coefficients = false; + global_flags.ycbcr_auto_coefficients = false; + } else { + fprintf(stderr, "ERROR: --output-ycbcr-coefficients must be “rec601”, “rec709” or “auto”\n"); + exit(1); + } + + if (global_flags.output_buffer_frames < 0.0f) { + // Actually, even zero probably won't make sense; there is some internal + // delay to the card. + fprintf(stderr, "ERROR: --output-buffer-frames can't be negative.\n"); + exit(1); + } + if (global_flags.output_slop_frames < 0.0f) { + fprintf(stderr, "ERROR: --output-slop-frames can't be negative.\n"); + exit(1); + } + if (global_flags.max_input_queue_frames < 1) { + fprintf(stderr, "ERROR: --max-input-queue-frames must be at least 1.\n"); + exit(1); + } + if (global_flags.max_input_queue_frames > 10) { + fprintf(stderr, "WARNING: --max-input-queue-frames has little effect over 10.\n"); + } + + if (!isinf(global_flags.x264_crf)) { // CRF mode is selected. + if (global_flags.x264_bitrate != -1) { + fprintf(stderr, "ERROR: --x264-bitrate and --x264-crf are mutually incompatible.\n"); + exit(1); + } + if (global_flags.x264_vbv_max_bitrate != -1 && global_flags.x264_vbv_buffer_size != -1) { + fprintf(stderr, "WARNING: VBV settings are ignored with --x264-crf.\n"); + } + } else if (global_flags.x264_bitrate == -1) { + global_flags.x264_bitrate = DEFAULT_X264_OUTPUT_BIT_RATE; + } +} diff --git a/nageru/flags.h b/nageru/flags.h new file mode 100644 index 0000000..09337d1 --- /dev/null +++ b/nageru/flags.h @@ -0,0 +1,80 @@ +#ifndef _FLAGS_H +#define _FLAGS_H + +#include + +#include +#include +#include + +#include "defs.h" +#include "ycbcr_interpretation.h" + +struct Flags { + int width = 1280, height = 720; + int num_cards = 2; + std::string va_display; + bool fake_cards_audio = false; + bool uncompressed_video_to_http = false; + bool x264_video_to_http = false; + bool x264_video_to_disk = false; // Disables Quick Sync entirely. Implies x264_video_to_http == true. + std::vector theme_dirs { ".", "/usr/local/share/nageru" }; + std::string recording_dir = "."; + std::string theme_filename = "theme.lua"; + bool locut_enabled = true; + bool gain_staging_auto = true; + float initial_gain_staging_db = 0.0f; + bool compressor_enabled = true; + bool limiter_enabled = true; + bool final_makeup_gain_auto = true; + bool flush_pbos = true; + std::string stream_mux_name = DEFAULT_STREAM_MUX_NAME; + bool stream_coarse_timebase = false; + std::string stream_audio_codec_name; // Blank = use the same as for the recording. + int stream_audio_codec_bitrate = DEFAULT_AUDIO_OUTPUT_BIT_RATE; // Ignored if stream_audio_codec_name is blank. + std::string x264_preset; // Empty will be overridden by X264_DEFAULT_PRESET, unless speedcontrol is set. + std::string x264_tune = X264_DEFAULT_TUNE; + bool x264_speedcontrol = false; + bool x264_speedcontrol_verbose = false; + int x264_bitrate = -1; // In kilobit/sec. -1 = not set = DEFAULT_X264_OUTPUT_BIT_RATE. + float x264_crf = HUGE_VAL; // From 51 - QP_MAX_SPEC to 51. HUGE_VAL = not set = use x264_bitrate instead. + int x264_vbv_max_bitrate = -1; // In kilobits. 0 = no limit, -1 = same as (CBR). + int x264_vbv_buffer_size = -1; // In kilobits. 0 = one-frame VBV, -1 = same as (one-second VBV). + std::vector x264_extra_param; // In “key[,value]” format. + bool enable_alsa_output = true; + std::map default_stream_mapping; + bool multichannel_mapping_mode = false; // Implicitly true if input_mapping_filename is nonempty. + std::string input_mapping_filename; // Empty for none. + std::string midi_mapping_filename; // Empty for none. + bool default_hdmi_input = false; + bool print_video_latency = false; + double audio_queue_length_ms = 100.0; + bool ycbcr_rec709_coefficients = false; // Will be overridden by HDMI/SDI output if ycbcr_auto_coefficients == true. + bool ycbcr_auto_coefficients = true; + int output_card = -1; + double output_buffer_frames = 6.0; + double output_slop_frames = 0.5; + int max_input_queue_frames = 6; + int http_port = DEFAULT_HTTPD_PORT; + bool display_timecode_in_stream = false; + bool display_timecode_on_stdout = false; + bool enable_quick_cut_keys = false; + bool ten_bit_input = false; + bool ten_bit_output = false; // Implies x264_video_to_disk == true and x264_bit_depth == 10. + YCbCrInterpretation ycbcr_interpretation[MAX_VIDEO_CARDS]; + bool transcode_audio = true; // Kaeru only. + int x264_bit_depth = 8; // Not user-settable. + bool use_zerocopy = false; // Not user-settable. + bool can_disable_srgb_decoder = false; // Not user-settable. + bool fullscreen = false; +}; +extern Flags global_flags; + +enum Program { + PROGRAM_NAGERU, + PROGRAM_KAERU +}; +void usage(Program program); +void parse_flags(Program program, int argc, char * const argv[]); + +#endif // !defined(_FLAGS_H) diff --git a/nageru/glwidget.cpp b/nageru/glwidget.cpp new file mode 100644 index 0000000..bf537de --- /dev/null +++ b/nageru/glwidget.cpp @@ -0,0 +1,436 @@ +#include "glwidget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "audio_mixer.h" +#include "context.h" +#include "context_menus.h" +#include "flags.h" +#include "mainwindow.h" +#include "mixer.h" +#include "ref_counted_gl_sync.h" + +class QMouseEvent; + +#undef Success +#include +#include + +using namespace movit; +using namespace std; +using namespace std::placeholders; + +namespace { + +double srgb_to_linear(double x) +{ + if (x < 0.04045) { + return x / 12.92; + } else { + return pow((x + 0.055) / 1.055, 2.4); + } +} + +} // namespace + +GLWidget::GLWidget(QWidget *parent) + : QGLWidget(parent, global_share_widget) +{ +} + +GLWidget::~GLWidget() +{ +} + +void GLWidget::shutdown() +{ + if (resource_pool != nullptr) { + makeCurrent(); + resource_pool->clean_context(); + } + global_mixer->remove_frame_ready_callback(output, this); +} + +void GLWidget::grab_white_balance(unsigned channel, unsigned x, unsigned y) +{ + // Set the white balance to neutral for the grab. It's probably going to + // flicker a bit, but hopefully this display is not live anyway. + global_mixer->set_wb(output, 0.5, 0.5, 0.5); + global_mixer->wait_for_next_frame(); + + // Mark that the next paintGL() should grab the given pixel. + grab_x = x; + grab_y = y; + grab_output = Mixer::Output(Mixer::OUTPUT_INPUT0 + channel); + should_grab = true; + + updateGL(); +} + +void GLWidget::initializeGL() +{ + static once_flag flag; + call_once(flag, [this]{ + global_mixer = new Mixer(QGLFormat::toSurfaceFormat(format()), global_flags.num_cards); + global_audio_mixer = global_mixer->get_audio_mixer(); + global_mainwindow->mixer_created(global_mixer); + global_mixer->start(); + }); + global_mixer->add_frame_ready_callback(output, this, [this]{ + QMetaObject::invokeMethod(this, "update", Qt::AutoConnection); + }); + if (output == Mixer::OUTPUT_LIVE) { + global_mixer->set_transition_names_updated_callback(output, [this](const vector &names){ + emit transition_names_updated(names); + }); + } + if (output >= Mixer::OUTPUT_INPUT0) { + global_mixer->set_name_updated_callback(output, [this](const string &name){ + emit name_updated(output, name); + }); + global_mixer->set_color_updated_callback(output, [this](const string &color){ + emit color_updated(output, color); + }); + } + setContextMenuPolicy(Qt::CustomContextMenu); + connect(this, &QWidget::customContextMenuRequested, bind(&GLWidget::show_context_menu, this, _1)); + + glDisable(GL_BLEND); + glDisable(GL_DEPTH_TEST); + glDepthMask(GL_FALSE); +} + +void GLWidget::resizeGL(int width, int height) +{ + current_width = width; + current_height = height; + glViewport(0, 0, width, height); +} + +void GLWidget::paintGL() +{ + Mixer::DisplayFrame frame; + if (!global_mixer->get_display_frame(output, &frame)) { + glClearColor(0.0f, 1.0f, 0.0f, 1.0f); + check_error(); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + check_error(); + return; + } + + check_error(); + glWaitSync(frame.ready_fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED); + check_error(); + frame.setup_chain(); + check_error(); + glDisable(GL_FRAMEBUFFER_SRGB); + check_error(); + frame.chain->render_to_fbo(0, current_width, current_height); + check_error(); + + if (resource_pool == nullptr) { + resource_pool = frame.chain->get_resource_pool(); + } else { + assert(resource_pool == frame.chain->get_resource_pool()); + } + + if (should_grab) { + GLfloat reference_color[4]; + glReadPixels(grab_x, current_height - grab_y - 1, 1, 1, GL_BGRA, GL_FLOAT, reference_color); + + double r = srgb_to_linear(reference_color[2]); + double g = srgb_to_linear(reference_color[1]); + double b = srgb_to_linear(reference_color[0]); + global_mixer->set_wb(grab_output, r, g, b); + should_grab = false; + } +} + +void GLWidget::mousePressEvent(QMouseEvent *event) +{ + emit clicked(); +} + +void GLWidget::show_context_menu(const QPoint &pos) +{ + if (output == Mixer::OUTPUT_LIVE) { + show_live_context_menu(pos); + } + if (output >= Mixer::OUTPUT_INPUT0) { + int signal_num = global_mixer->get_channel_signal(output); + if (signal_num != -1) { + show_preview_context_menu(signal_num, pos); + } + } +} + +void GLWidget::show_live_context_menu(const QPoint &pos) +{ + QPoint global_pos = mapToGlobal(pos); + + QMenu menu; + + // Add a submenu for selecting output card, with an action for each card. + QMenu card_submenu; + fill_hdmi_sdi_output_device_menu(&card_submenu); + card_submenu.setTitle("HDMI/SDI output device"); + menu.addMenu(&card_submenu); + + // Add a submenu for choosing the output resolution. Since this is + // card-dependent, it is disabled if we haven't chosen a card + // (but it's still there so that the user will know it exists). + QMenu resolution_submenu; + fill_hdmi_sdi_output_resolution_menu(&resolution_submenu); + resolution_submenu.setTitle("HDMI/SDI output resolution"); + menu.addMenu(&resolution_submenu); + + // Show the menu; if there's an action selected, it will deal with it itself. + menu.exec(global_pos); +} + +void GLWidget::show_preview_context_menu(unsigned signal_num, const QPoint &pos) +{ + QPoint global_pos = mapToGlobal(pos); + + QMenu menu; + + // Add a submenu for selecting input card, with an action for each card. + QMenu card_submenu; + QActionGroup card_group(&card_submenu); + + QMenu interpretation_submenu; + QActionGroup interpretation_group(&interpretation_submenu); + + QMenu video_input_submenu; + QActionGroup video_input_group(&video_input_submenu); + + QMenu audio_input_submenu; + QActionGroup audio_input_group(&audio_input_submenu); + + QMenu mode_submenu; + QActionGroup mode_group(&mode_submenu); + + unsigned num_cards = global_mixer->get_num_cards(); + unsigned current_card = global_mixer->map_signal(signal_num); + bool is_ffmpeg = global_mixer->card_is_ffmpeg(current_card); + + if (!is_ffmpeg) { // FFmpeg inputs are not connected to any card; they're locked to a given input and have a given Y'CbCr interpretatio and have a given Y'CbCr interpretationn. + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + QString description(QString::fromStdString(global_mixer->get_card_description(card_index))); + QAction *action = new QAction(description, &card_group); + action->setCheckable(true); + if (current_card == card_index) { + action->setChecked(true); + } + action->setData(QList{"card", card_index}); + card_submenu.addAction(action); + } + + card_submenu.setTitle("Input source"); + menu.addMenu(&card_submenu); + + // Note that this setting depends on which card is active. + + YCbCrInterpretation current_interpretation = global_mixer->get_input_ycbcr_interpretation(current_card); + { + QAction *action = new QAction("Auto", &interpretation_group); + action->setCheckable(true); + if (current_interpretation.ycbcr_coefficients_auto) { + action->setChecked(true); + } + action->setData(QList{"interpretation", true, YCBCR_REC_709, false}); + interpretation_submenu.addAction(action); + } + for (YCbCrLumaCoefficients ycbcr_coefficients : { YCBCR_REC_709, YCBCR_REC_601 }) { + for (bool full_range : { false, true }) { + std::string description; + if (ycbcr_coefficients == YCBCR_REC_709) { + description = "Rec. 709 (HD)"; + } else { + description = "Rec. 601 (SD)"; + } + if (full_range) { + description += ", full range (nonstandard)"; + } + QAction *action = new QAction(QString::fromStdString(description), &interpretation_group); + action->setCheckable(true); + if (!current_interpretation.ycbcr_coefficients_auto && + ycbcr_coefficients == current_interpretation.ycbcr_coefficients && + full_range == current_interpretation.full_range) { + action->setChecked(true); + } + action->setData(QList{"interpretation", false, ycbcr_coefficients, full_range}); + interpretation_submenu.addAction(action); + } + } + + interpretation_submenu.setTitle("Input interpretation"); + menu.addMenu(&interpretation_submenu); + } + + // --- The choices in the next few options depend a lot on which card is active --- + + bool has_auto_mode = false; + QAction *change_url_action = nullptr; + if (is_ffmpeg) { + // Add a menu to change the source URL if we're an FFmpeg card. + // (The theme can still override.) + if (global_mixer->card_is_ffmpeg(current_card)) { + change_url_action = new QAction("Change source filename/URL…", &menu); + menu.addAction(change_url_action); + } + } else { + // Add a submenu for selecting video input, with an action for each input. + std::map video_inputs = global_mixer->get_available_video_inputs(current_card); + uint32_t current_video_input = global_mixer->get_current_video_input(current_card); + for (const auto &mode : video_inputs) { + QString description(QString::fromStdString(mode.second)); + QAction *action = new QAction(description, &video_input_group); + action->setCheckable(true); + if (mode.first == current_video_input) { + action->setChecked(true); + } + action->setData(QList{"video_input", mode.first}); + video_input_submenu.addAction(action); + } + + video_input_submenu.setTitle("Video input"); + menu.addMenu(&video_input_submenu); + + // The same for audio input. + std::map audio_inputs = global_mixer->get_available_audio_inputs(current_card); + uint32_t current_audio_input = global_mixer->get_current_audio_input(current_card); + for (const auto &mode : audio_inputs) { + QString description(QString::fromStdString(mode.second)); + QAction *action = new QAction(description, &audio_input_group); + action->setCheckable(true); + if (mode.first == current_audio_input) { + action->setChecked(true); + } + action->setData(QList{"audio_input", mode.first}); + audio_input_submenu.addAction(action); + } + + audio_input_submenu.setTitle("Audio input"); + menu.addMenu(&audio_input_submenu); + + // The same for resolution. + std::map video_modes = global_mixer->get_available_video_modes(current_card); + uint32_t current_video_mode = global_mixer->get_current_video_mode(current_card); + for (const auto &mode : video_modes) { + QString description(QString::fromStdString(mode.second.name)); + QAction *action = new QAction(description, &mode_group); + action->setCheckable(true); + if (mode.first == current_video_mode) { + action->setChecked(true); + } + action->setData(QList{"video_mode", mode.first}); + mode_submenu.addAction(action); + + // TODO: Relying on the 0 value here (from bmusb.h) is ugly, it should be a named constant. + if (mode.first == 0) { + has_auto_mode = true; + } + } + + // Add a “scan” menu if there's no “auto” mode. + if (!has_auto_mode) { + QAction *action = new QAction("Scan", &mode_group); + action->setData(QList{"video_mode", 0}); + mode_submenu.addSeparator(); + mode_submenu.addAction(action); + } + + mode_submenu.setTitle("Input mode"); + menu.addMenu(&mode_submenu); + } + + // --- End of card-dependent choices --- + + // Add an audio source selector. + QAction *audio_source_action = nullptr; + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::SIMPLE) { + audio_source_action = new QAction("Use as audio source", &menu); + audio_source_action->setCheckable(true); + if (global_audio_mixer->get_simple_input() == current_card) { + audio_source_action->setChecked(true); + audio_source_action->setEnabled(false); + } + menu.addAction(audio_source_action); + } + + // And a master clock selector. + QAction *master_clock_action = new QAction("Use as master clock", &menu); + master_clock_action->setCheckable(true); + if (global_mixer->get_output_card_index() != -1) { + master_clock_action->setChecked(false); + master_clock_action->setEnabled(false); + } else if (global_mixer->get_master_clock() == signal_num) { + master_clock_action->setChecked(true); + master_clock_action->setEnabled(false); + } + menu.addAction(master_clock_action); + + // Show the menu and look at the result. + QAction *selected_item = menu.exec(global_pos); + if (audio_source_action != nullptr && selected_item == audio_source_action) { + global_audio_mixer->set_simple_input(current_card); + } else if (change_url_action != nullptr && selected_item == change_url_action) { + // NOTE: We can't use “this” as parent, since the dialog would inherit our style sheet. + bool ok; + const string url = global_mixer->get_ffmpeg_filename(current_card); + QString new_url = QInputDialog::getText(window(), tr("Change URL"), + tr("Enter new filename/URL for the given video input:"), QLineEdit::Normal, + QString::fromStdString(url), &ok); + // FIXME prefill the input + if (ok) { + global_mixer->set_ffmpeg_filename(current_card, new_url.toStdString()); + } + } else if (selected_item == master_clock_action) { + global_mixer->set_master_clock(signal_num); + } else if (selected_item != nullptr) { + QList selected = selected_item->data().toList(); + if (selected[0].toString() == "video_mode") { + uint32_t mode = selected[1].toUInt(nullptr); + if (mode == 0 && !has_auto_mode) { + global_mixer->start_mode_scanning(current_card); + } else { + global_mixer->set_video_mode(current_card, mode); + } + } else if (selected[0].toString() == "video_input") { + uint32_t input = selected[1].toUInt(nullptr); + global_mixer->set_video_input(current_card, input); + } else if (selected[0].toString() == "audio_input") { + uint32_t input = selected[1].toUInt(nullptr); + global_mixer->set_audio_input(current_card, input); + } else if (selected[0].toString() == "card") { + unsigned card_index = selected[1].toUInt(nullptr); + global_mixer->set_signal_mapping(signal_num, card_index); + } else if (selected[0].toString() == "interpretation") { + YCbCrInterpretation interpretation; + interpretation.ycbcr_coefficients_auto = selected[1].toBool(); + interpretation.ycbcr_coefficients = YCbCrLumaCoefficients(selected[2].toUInt(nullptr)); + interpretation.full_range = selected[3].toBool(); + global_mixer->set_input_ycbcr_interpretation(current_card, interpretation); + } else { + assert(false); + } + } +} diff --git a/nageru/glwidget.h b/nageru/glwidget.h new file mode 100644 index 0000000..9b554d0 --- /dev/null +++ b/nageru/glwidget.h @@ -0,0 +1,74 @@ +#ifndef GLWIDGET_H +#define GLWIDGET_H + +#include +#include +#include +#include +#include + +#include "mixer.h" + +class QMouseEvent; +class QObject; +class QPoint; +class QWidget; + +namespace movit { + +class ResourcePool; + +} // namespace movit + +// Note: We use the older QGLWidget instead of QOpenGLWidget as it is +// much faster (does not go through a separate offscreen rendering step). +// +// TODO: Consider if QOpenGLWindow could do what we want. +class GLWidget : public QGLWidget +{ + Q_OBJECT + +public: + GLWidget(QWidget *parent = 0); + ~GLWidget(); + + void set_output(Mixer::Output output) + { + this->output = output; + } + + void shutdown(); + + // NOTE: Will make the white balance flicker for a frame. + void grab_white_balance(unsigned channel, unsigned x, unsigned y); + +protected: + void initializeGL() override; + void resizeGL(int width, int height) override; + void paintGL() override; + void mousePressEvent(QMouseEvent *event) override; + +signals: + void clicked(); + void transition_names_updated(std::vector transition_names); + void name_updated(Mixer::Output output, const std::string &name); + void color_updated(Mixer::Output output, const std::string &color); + +private slots: + void show_context_menu(const QPoint &pos); + +private: + void show_live_context_menu(const QPoint &pos); + void show_preview_context_menu(unsigned signal_num, const QPoint &pos); + + Mixer::Output output; + GLuint vao, program_num; + GLuint position_vbo, texcoord_vbo; + movit::ResourcePool *resource_pool = nullptr; + int current_width = 1, current_height = 1; + bool should_grab = false; + unsigned grab_x, grab_y; + Mixer::Output grab_output; // Should nominally be the same as output. +}; + +#endif diff --git a/nageru/httpd.cpp b/nageru/httpd.cpp new file mode 100644 index 0000000..f644176 --- /dev/null +++ b/nageru/httpd.cpp @@ -0,0 +1,275 @@ +#include "httpd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} + +#include "defs.h" +#include "metacube2.h" +#include "metrics.h" + +struct MHD_Connection; +struct MHD_Response; + +using namespace std; + +HTTPD::HTTPD() +{ + global_metrics.add("num_connected_clients", &metric_num_connected_clients, Metrics::TYPE_GAUGE); +} + +HTTPD::~HTTPD() +{ + stop(); +} + +void HTTPD::start(int port) +{ + mhd = MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_POLL_INTERNALLY | MHD_USE_DUAL_STACK, + port, + nullptr, nullptr, + &answer_to_connection_thunk, this, + MHD_OPTION_NOTIFY_COMPLETED, nullptr, this, + MHD_OPTION_END); + if (mhd == nullptr) { + fprintf(stderr, "Warning: Could not open HTTP server. (Port already in use?)\n"); + } +} + +void HTTPD::stop() +{ + if (mhd) { + MHD_quiesce_daemon(mhd); + for (Stream *stream : streams) { + stream->stop(); + } + MHD_stop_daemon(mhd); + mhd = nullptr; + } +} + +void HTTPD::add_data(const char *buf, size_t size, bool keyframe, int64_t time, AVRational timebase) +{ + unique_lock lock(streams_mutex); + for (Stream *stream : streams) { + stream->add_data(buf, size, keyframe ? Stream::DATA_TYPE_KEYFRAME : Stream::DATA_TYPE_OTHER, time, timebase); + } +} + +int HTTPD::answer_to_connection_thunk(void *cls, MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls) +{ + HTTPD *httpd = (HTTPD *)cls; + return httpd->answer_to_connection(connection, url, method, version, upload_data, upload_data_size, con_cls); +} + +int HTTPD::answer_to_connection(MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls) +{ + // See if the URL ends in “.metacube”. + HTTPD::Stream::Framing framing; + if (strstr(url, ".metacube") == url + strlen(url) - strlen(".metacube")) { + framing = HTTPD::Stream::FRAMING_METACUBE; + } else { + framing = HTTPD::Stream::FRAMING_RAW; + } + + if (strcmp(url, "/metrics") == 0) { + string contents = global_metrics.serialize(); + MHD_Response *response = MHD_create_response_from_buffer( + contents.size(), &contents[0], MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(response, "Content-type", "text/plain"); + int ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + MHD_destroy_response(response); // Only decreases the refcount; actual free is after the request is done. + return ret; + } + if (endpoints.count(url)) { + pair contents_and_type = endpoints[url].callback(); + MHD_Response *response = MHD_create_response_from_buffer( + contents_and_type.first.size(), &contents_and_type.first[0], MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(response, "Content-type", contents_and_type.second.c_str()); + if (endpoints[url].cors_policy == ALLOW_ALL_ORIGINS) { + MHD_add_response_header(response, "Access-Control-Allow-Origin", "*"); + } + int ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + MHD_destroy_response(response); // Only decreases the refcount; actual free is after the request is done. + return ret; + } + + // Small hack; reject unknown /channels/foo. + if (string(url).find("/channels/") == 0) { + string contents = "Not found."; + MHD_Response *response = MHD_create_response_from_buffer( + contents.size(), &contents[0], MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(response, "Content-type", "text/plain"); + int ret = MHD_queue_response(connection, MHD_HTTP_NOT_FOUND, response); + MHD_destroy_response(response); // Only decreases the refcount; actual free is after the request is done. + return ret; + } + + HTTPD::Stream *stream = new HTTPD::Stream(this, framing); + stream->add_data(header.data(), header.size(), Stream::DATA_TYPE_HEADER, AV_NOPTS_VALUE, AVRational{ 1, 0 }); + { + unique_lock lock(streams_mutex); + streams.insert(stream); + } + ++metric_num_connected_clients; + *con_cls = stream; + + // Does not strictly have to be equal to MUX_BUFFER_SIZE. + MHD_Response *response = MHD_create_response_from_callback( + (size_t)-1, MUX_BUFFER_SIZE, &HTTPD::Stream::reader_callback_thunk, stream, &HTTPD::free_stream); + // TODO: Content-type? + if (framing == HTTPD::Stream::FRAMING_METACUBE) { + MHD_add_response_header(response, "Content-encoding", "metacube"); + } + + int ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + MHD_destroy_response(response); // Only decreases the refcount; actual free is after the request is done. + + return ret; +} + +void HTTPD::free_stream(void *cls) +{ + HTTPD::Stream *stream = (HTTPD::Stream *)cls; + HTTPD *httpd = stream->get_parent(); + { + unique_lock lock(httpd->streams_mutex); + delete stream; + httpd->streams.erase(stream); + } + --httpd->metric_num_connected_clients; +} + +ssize_t HTTPD::Stream::reader_callback_thunk(void *cls, uint64_t pos, char *buf, size_t max) +{ + HTTPD::Stream *stream = (HTTPD::Stream *)cls; + return stream->reader_callback(pos, buf, max); +} + +ssize_t HTTPD::Stream::reader_callback(uint64_t pos, char *buf, size_t max) +{ + unique_lock lock(buffer_mutex); + has_buffered_data.wait(lock, [this]{ return should_quit || !buffered_data.empty(); }); + if (should_quit) { + return 0; + } + + ssize_t ret = 0; + while (max > 0 && !buffered_data.empty()) { + const string &s = buffered_data.front(); + assert(s.size() > used_of_buffered_data); + size_t len = s.size() - used_of_buffered_data; + if (max >= len) { + // Consume the entire (rest of the) string. + memcpy(buf, s.data() + used_of_buffered_data, len); + buf += len; + ret += len; + max -= len; + buffered_data.pop_front(); + used_of_buffered_data = 0; + } else { + // We don't need the entire string; just use the first part of it. + memcpy(buf, s.data() + used_of_buffered_data, max); + buf += max; + used_of_buffered_data += max; + ret += max; + max = 0; + } + } + + return ret; +} + +void HTTPD::Stream::add_data(const char *buf, size_t buf_size, HTTPD::Stream::DataType data_type, int64_t time, AVRational timebase) +{ + if (buf_size == 0) { + return; + } + if (data_type == DATA_TYPE_KEYFRAME) { + seen_keyframe = true; + } else if (data_type == DATA_TYPE_OTHER && !seen_keyframe) { + // Start sending only once we see a keyframe. + return; + } + + unique_lock lock(buffer_mutex); + + if (framing == FRAMING_METACUBE) { + int flags = 0; + if (data_type == DATA_TYPE_HEADER) { + flags |= METACUBE_FLAGS_HEADER; + } else if (data_type == DATA_TYPE_OTHER) { + flags |= METACUBE_FLAGS_NOT_SUITABLE_FOR_STREAM_START; + } + + // If we're about to send a keyframe, send a pts metadata block + // to mark its time. + if ((flags & METACUBE_FLAGS_NOT_SUITABLE_FOR_STREAM_START) == 0 && time != AV_NOPTS_VALUE) { + metacube2_pts_packet packet; + packet.type = htobe64(METACUBE_METADATA_TYPE_NEXT_BLOCK_PTS); + packet.pts = htobe64(time); + packet.timebase_num = htobe64(timebase.num); + packet.timebase_den = htobe64(timebase.den); + + metacube2_block_header hdr; + memcpy(hdr.sync, METACUBE2_SYNC, sizeof(hdr.sync)); + hdr.size = htonl(sizeof(packet)); + hdr.flags = htons(METACUBE_FLAGS_METADATA); + hdr.csum = htons(metacube2_compute_crc(&hdr)); + buffered_data.emplace_back((char *)&hdr, sizeof(hdr)); + buffered_data.emplace_back((char *)&packet, sizeof(packet)); + } + + metacube2_block_header hdr; + memcpy(hdr.sync, METACUBE2_SYNC, sizeof(hdr.sync)); + hdr.size = htonl(buf_size); + hdr.flags = htons(flags); + hdr.csum = htons(metacube2_compute_crc(&hdr)); + buffered_data.emplace_back((char *)&hdr, sizeof(hdr)); + } + buffered_data.emplace_back(buf, buf_size); + + // Send a Metacube2 timestamp every keyframe. + if (framing == FRAMING_METACUBE && data_type == DATA_TYPE_KEYFRAME) { + timespec now; + clock_gettime(CLOCK_REALTIME, &now); + + metacube2_timestamp_packet packet; + packet.type = htobe64(METACUBE_METADATA_TYPE_ENCODER_TIMESTAMP); + packet.tv_sec = htobe64(now.tv_sec); + packet.tv_nsec = htobe64(now.tv_nsec); + + metacube2_block_header hdr; + memcpy(hdr.sync, METACUBE2_SYNC, sizeof(hdr.sync)); + hdr.size = htonl(sizeof(packet)); + hdr.flags = htons(METACUBE_FLAGS_METADATA); + hdr.csum = htons(metacube2_compute_crc(&hdr)); + buffered_data.emplace_back((char *)&hdr, sizeof(hdr)); + buffered_data.emplace_back((char *)&packet, sizeof(packet)); + } + + has_buffered_data.notify_all(); +} + +void HTTPD::Stream::stop() +{ + unique_lock lock(buffer_mutex); + should_quit = true; + has_buffered_data.notify_all(); +} diff --git a/nageru/httpd.h b/nageru/httpd.h new file mode 100644 index 0000000..57c649b --- /dev/null +++ b/nageru/httpd.h @@ -0,0 +1,115 @@ +#ifndef _HTTPD_H +#define _HTTPD_H + +// A class dealing with stream output to HTTP. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +struct MHD_Connection; +struct MHD_Daemon; + +class HTTPD { +public: + // Returns a pair of content and content-type. + using EndpointCallback = std::function()>; + + HTTPD(); + ~HTTPD(); + + // Should be called before start(). + void set_header(const std::string &data) { + header = data; + } + + // Should be called before start() (due to threading issues). + enum CORSPolicy { + NO_CORS_POLICY, + ALLOW_ALL_ORIGINS + }; + void add_endpoint(const std::string &url, const EndpointCallback &callback, CORSPolicy cors_policy) { + endpoints[url] = Endpoint{ callback, cors_policy }; + } + + void start(int port); + void stop(); + void add_data(const char *buf, size_t size, bool keyframe, int64_t time, AVRational timebase); + int64_t get_num_connected_clients() const { + return metric_num_connected_clients.load(); + } + +private: + static int answer_to_connection_thunk(void *cls, MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls); + + int answer_to_connection(MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls); + + static void free_stream(void *cls); + + + class Stream { + public: + enum Framing { + FRAMING_RAW, + FRAMING_METACUBE + }; + Stream(HTTPD *parent, Framing framing) : parent(parent), framing(framing) {} + + static ssize_t reader_callback_thunk(void *cls, uint64_t pos, char *buf, size_t max); + ssize_t reader_callback(uint64_t pos, char *buf, size_t max); + + enum DataType { + DATA_TYPE_HEADER, + DATA_TYPE_KEYFRAME, + DATA_TYPE_OTHER + }; + void add_data(const char *buf, size_t size, DataType data_type, int64_t time, AVRational timebase); + void stop(); + HTTPD *get_parent() const { return parent; } + + private: + HTTPD *parent; + Framing framing; + + std::mutex buffer_mutex; + bool should_quit = false; // Under . + std::condition_variable has_buffered_data; + std::deque buffered_data; // Protected by . + size_t used_of_buffered_data = 0; // How many bytes of the first element of that is already used. Protected by . + size_t seen_keyframe = false; + }; + + MHD_Daemon *mhd = nullptr; + std::mutex streams_mutex; + std::set streams; // Not owned. + struct Endpoint { + EndpointCallback callback; + CORSPolicy cors_policy; + }; + std::unordered_map endpoints; + std::string header; + + // Metrics. + std::atomic metric_num_connected_clients{0}; +}; + +#endif // !defined(_HTTPD_H) diff --git a/nageru/image_input.cpp b/nageru/image_input.cpp new file mode 100644 index 0000000..2bf4a23 --- /dev/null +++ b/nageru/image_input.cpp @@ -0,0 +1,260 @@ +#include "image_input.h" + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +#include +} + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ffmpeg_raii.h" +#include "ffmpeg_util.h" +#include "flags.h" + +struct SwsContext; + +using namespace std; + +ImageInput::ImageInput(const string &filename) + : movit::FlatInput({movit::COLORSPACE_sRGB, movit::GAMMA_sRGB}, movit::FORMAT_RGBA_POSTMULTIPLIED_ALPHA, + GL_UNSIGNED_BYTE, 1280, 720), // Resolution will be overwritten. + filename(filename), + pathname(search_for_file_or_die(filename)), + current_image(load_image(filename, pathname)) +{ + if (current_image == nullptr) { // Could happen even though search_for_file() returned. + fprintf(stderr, "Couldn't load image, exiting.\n"); + exit(1); + } + set_width(current_image->width); + set_height(current_image->height); + set_pixel_data(current_image->pixels.get()); +} + +void ImageInput::set_gl_state(GLuint glsl_program_num, const string& prefix, unsigned *sampler_num) +{ + // See if the background thread has given us a new version of our image. + // Note: The old version might still be lying around in other ImageInputs + // (in fact, it's likely), but at least the total amount of memory used + // is bounded. Currently we don't even share textures between them, + // so there's a fair amount of OpenGL memory waste anyway (the cache + // is mostly there to save startup time, not RAM). + { + unique_lock lock(all_images_lock); + if (all_images[pathname] != current_image) { + current_image = all_images[pathname]; + set_pixel_data(current_image->pixels.get()); + } + } + movit::FlatInput::set_gl_state(glsl_program_num, prefix, sampler_num); +} + +shared_ptr ImageInput::load_image(const string &filename, const string &pathname) +{ + unique_lock lock(all_images_lock); // Held also during loading. + if (all_images.count(pathname)) { + return all_images[pathname]; + } + + all_images[pathname] = load_image_raw(pathname); + timespec first_modified = all_images[pathname]->last_modified; + update_threads[pathname] = + thread(bind(update_thread_func, filename, pathname, first_modified)); + + return all_images[pathname]; +} + +shared_ptr ImageInput::load_image_raw(const string &pathname) +{ + // Note: Call before open, not after; otherwise, there's a race. + // (There is now, too, but it tips the correct way. We could use fstat() + // if we had the file descriptor.) + struct stat buf; + if (stat(pathname.c_str(), &buf) != 0) { + fprintf(stderr, "%s: Error stat-ing file\n", pathname.c_str()); + return nullptr; + } + timespec last_modified = buf.st_mtim; + + auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, nullptr); + if (format_ctx == nullptr) { + fprintf(stderr, "%s: Error opening file\n", pathname.c_str()); + return nullptr; + } + + if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) { + fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str()); + return nullptr; + } + + int stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_VIDEO); + if (stream_index == -1) { + fprintf(stderr, "%s: No video stream found\n", pathname.c_str()); + return nullptr; + } + + const AVCodecParameters *codecpar = format_ctx->streams[stream_index]->codecpar; + AVCodecContextWithDeleter codec_ctx = avcodec_alloc_context3_unique(nullptr); + if (avcodec_parameters_to_context(codec_ctx.get(), codecpar) < 0) { + fprintf(stderr, "%s: Cannot fill codec parameters\n", pathname.c_str()); + return nullptr; + } + AVCodec *codec = avcodec_find_decoder(codecpar->codec_id); + if (codec == nullptr) { + fprintf(stderr, "%s: Cannot find decoder\n", pathname.c_str()); + return nullptr; + } + if (avcodec_open2(codec_ctx.get(), codec, nullptr) < 0) { + fprintf(stderr, "%s: Cannot open decoder\n", pathname.c_str()); + return nullptr; + } + unique_ptr codec_ctx_cleanup( + codec_ctx.get(), avcodec_close); + + // Read packets until we have a frame or there are none left. + int frame_finished = 0; + AVFrameWithDeleter frame = av_frame_alloc_unique(); + bool eof = false; + do { + AVPacket pkt; + unique_ptr pkt_cleanup( + &pkt, av_packet_unref); + av_init_packet(&pkt); + pkt.data = nullptr; + pkt.size = 0; + if (av_read_frame(format_ctx.get(), &pkt) == 0) { + if (pkt.stream_index != stream_index) { + continue; + } + if (avcodec_send_packet(codec_ctx.get(), &pkt) < 0) { + fprintf(stderr, "%s: Cannot send packet to codec.\n", pathname.c_str()); + return nullptr; + } + } else { + eof = true; // Or error, but ignore that for the time being. + } + + int err = avcodec_receive_frame(codec_ctx.get(), frame.get()); + if (err == 0) { + frame_finished = true; + break; + } else if (err != AVERROR(EAGAIN)) { + fprintf(stderr, "%s: Cannot receive frame from codec.\n", pathname.c_str()); + return nullptr; + } + } while (!eof); + + if (!frame_finished) { + fprintf(stderr, "%s: Decoder did not output frame.\n", pathname.c_str()); + return nullptr; + } + + uint8_t *pic_data[4] = {nullptr}; + unique_ptr pic_data_cleanup( + &pic_data[0], av_freep); + int linesizes[4]; + if (av_image_alloc(pic_data, linesizes, frame->width, frame->height, AV_PIX_FMT_RGBA, 1) < 0) { + fprintf(stderr, "%s: Could not allocate picture data\n", pathname.c_str()); + return nullptr; + } + unique_ptr sws_ctx( + sws_getContext(frame->width, frame->height, + (AVPixelFormat)frame->format, frame->width, frame->height, + AV_PIX_FMT_RGBA, SWS_BICUBIC, nullptr, nullptr, nullptr), + sws_freeContext); + if (sws_ctx == nullptr) { + fprintf(stderr, "%s: Could not create scaler context\n", pathname.c_str()); + return nullptr; + } + sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes); + + size_t len = frame->width * frame->height * 4; + unique_ptr image_data(new uint8_t[len]); + av_image_copy_to_buffer(image_data.get(), len, pic_data, linesizes, AV_PIX_FMT_RGBA, frame->width, frame->height, 1); + + shared_ptr image(new Image{unsigned(frame->width), unsigned(frame->height), move(image_data), last_modified}); + return image; +} + +// Fire up a thread to update the image every second. +// We could do inotify, but this is good enough for now. +void ImageInput::update_thread_func(const std::string &filename, const std::string &pathname, const timespec &first_modified) +{ + char thread_name[16]; + snprintf(thread_name, sizeof(thread_name), "Update_%s", filename.c_str()); + pthread_setname_np(pthread_self(), thread_name); + + timespec last_modified = first_modified; + struct stat buf; + for ( ;; ) { + { + unique_lock lock(threads_should_quit_mu); + threads_should_quit_modified.wait_for(lock, chrono::seconds(1), []() { return threads_should_quit; }); + } + + if (threads_should_quit) { + return; + } + + if (stat(pathname.c_str(), &buf) != 0) { + fprintf(stderr, "%s: Couldn't check for new version, leaving the old in place.\n", pathname.c_str()); + continue; + } + if (buf.st_mtim.tv_sec == last_modified.tv_sec && + buf.st_mtim.tv_nsec == last_modified.tv_nsec) { + // Not changed. + continue; + } + shared_ptr image = load_image_raw(pathname); + if (image == nullptr) { + fprintf(stderr, "Couldn't load image, leaving the old in place.\n"); + continue; + } + fprintf(stderr, "Loaded new version of %s from disk.\n", pathname.c_str()); + unique_lock lock(all_images_lock); + all_images[pathname] = image; + last_modified = image->last_modified; + } +} + +void ImageInput::shutdown_updaters() +{ + { + unique_lock lock(threads_should_quit_mu); + threads_should_quit = true; + threads_should_quit_modified.notify_all(); + } + for (auto &it : update_threads) { + it.second.join(); + } +} + +mutex ImageInput::all_images_lock; +map> ImageInput::all_images; +map ImageInput::update_threads; +mutex ImageInput::threads_should_quit_mu; +bool ImageInput::threads_should_quit = false; +condition_variable ImageInput::threads_should_quit_modified; diff --git a/nageru/image_input.h b/nageru/image_input.h new file mode 100644 index 0000000..02be497 --- /dev/null +++ b/nageru/image_input.h @@ -0,0 +1,49 @@ +#ifndef _IMAGE_INPUT_H +#define _IMAGE_INPUT_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// An output that takes its input from a static image, loaded with ffmpeg. +// comes from a single 2D array with chunky pixels. The image is refreshed +// from disk about every second. +class ImageInput : public movit::FlatInput { +public: + ImageInput(const std::string &filename); + + std::string effect_type_id() const override { return "ImageInput"; } + void set_gl_state(GLuint glsl_program_num, const std::string& prefix, unsigned *sampler_num) override; + static void shutdown_updaters(); + +private: + struct Image { + unsigned width, height; + std::unique_ptr pixels; + timespec last_modified; + }; + + std::string filename, pathname; + std::shared_ptr current_image; + + static std::shared_ptr load_image(const std::string &filename, const std::string &pathname); + static std::shared_ptr load_image_raw(const std::string &pathname); + static void update_thread_func(const std::string &filename, const std::string &pathname, const timespec &first_modified); + static std::mutex all_images_lock; + static std::map> all_images; + static std::map update_threads; + + static std::mutex threads_should_quit_mu; + static bool threads_should_quit; // Under threads_should_quit_mu. + static std::condition_variable threads_should_quit_modified; // Signals when threads_should_quit is set. +}; + +#endif // !defined(_IMAGE_INPUT_H) diff --git a/nageru/input_mapping.cpp b/nageru/input_mapping.cpp new file mode 100644 index 0000000..45b6009 --- /dev/null +++ b/nageru/input_mapping.cpp @@ -0,0 +1,216 @@ +#include "input_mapping.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "audio_mixer.h" +#include "state.pb.h" + +using namespace std; +using namespace google::protobuf; + +string spec_to_string(DeviceSpec device_spec) +{ + char buf[256]; + + switch (device_spec.type) { + case InputSourceType::SILENCE: + return ""; + case InputSourceType::CAPTURE_CARD: + snprintf(buf, sizeof(buf), "Capture card %u", device_spec.index); + return buf; + case InputSourceType::ALSA_INPUT: + snprintf(buf, sizeof(buf), "ALSA input %u", device_spec.index); + return buf; + case InputSourceType::FFMPEG_VIDEO_INPUT: + snprintf(buf, sizeof(buf), "FFmpeg input %u", device_spec.index); + return buf; + default: + assert(false); + } +} + +bool save_input_mapping_to_file(const map &devices, const InputMapping &input_mapping, const string &filename) +{ + InputMappingProto mapping_proto; + { + map used_devices; + for (const InputMapping::Bus &bus : input_mapping.buses) { + if (!used_devices.count(bus.device)) { + used_devices.emplace(bus.device, used_devices.size()); + global_audio_mixer->serialize_device(bus.device, mapping_proto.add_device()); + } + + BusProto *bus_proto = mapping_proto.add_bus(); + bus_proto->set_name(bus.name); + bus_proto->set_device_index(used_devices[bus.device]); + bus_proto->set_source_channel_left(bus.source_channel[0]); + bus_proto->set_source_channel_right(bus.source_channel[1]); + } + } + + // Save to disk. We use the text format because it's friendlier + // for a user to look at and edit. + int fd = open(filename.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd == -1) { + perror(filename.c_str()); + return false; + } + io::FileOutputStream output(fd); // Takes ownership of fd. + if (!TextFormat::Print(mapping_proto, &output)) { + // TODO: Don't overwrite the old file (if any) on error. + output.Close(); + return false; + } + + output.Close(); + return true; +} + +bool load_input_mapping_from_file(const map &devices, const string &filename, InputMapping *new_mapping) +{ + // Read and parse the protobuf from disk. + int fd = open(filename.c_str(), O_RDONLY); + if (fd == -1) { + perror(filename.c_str()); + return false; + } + io::FileInputStream input(fd); // Takes ownership of fd. + InputMappingProto mapping_proto; + if (!TextFormat::Parse(&input, &mapping_proto)) { + input.Close(); + return false; + } + input.Close(); + + // Map devices in the proto to our current ones: + + // Get a list of all active devices. + set remaining_devices; + for (const auto &device_spec_and_info : devices) { + remaining_devices.insert(device_spec_and_info.first); + } + + // Now look at every device in the serialized protobuf and try to map + // it to one device we haven't taken yet. This isn't a full maximal matching, + // but it's good enough for our uses. + vector device_mapping; + for (unsigned device_index = 0; device_index < unsigned(mapping_proto.device_size()); ++device_index) { + const DeviceSpecProto &device_proto = mapping_proto.device(device_index); + switch (device_proto.type()) { + case DeviceSpecProto::SILENCE: + device_mapping.push_back(DeviceSpec{InputSourceType::SILENCE, 0}); + break; + case DeviceSpecProto::FFMPEG_VIDEO_INPUT: + case DeviceSpecProto::CAPTURE_CARD: { + // First see if there's a card that matches on both index and name. + DeviceSpec spec; + spec.type = (device_proto.type() == DeviceSpecProto::CAPTURE_CARD) ? + InputSourceType::CAPTURE_CARD : InputSourceType::FFMPEG_VIDEO_INPUT; + spec.index = unsigned(device_proto.index()); + assert(devices.count(spec)); + + const DeviceInfo &dev = devices.find(spec)->second; + if (remaining_devices.count(spec) && + dev.display_name == device_proto.display_name()) { + device_mapping.push_back(spec); + remaining_devices.erase(spec); + goto found_capture_card; + } + + // Scan and see if there's a match on name alone. + for (const DeviceSpec &spec : remaining_devices) { + if (spec.type == InputSourceType::CAPTURE_CARD && + dev.display_name == device_proto.display_name()) { + device_mapping.push_back(spec); + remaining_devices.erase(spec); + goto found_capture_card; + } + } + + // OK, see if at least the index is free. + if (remaining_devices.count(spec)) { + device_mapping.push_back(spec); + remaining_devices.erase(spec); + goto found_capture_card; + } + + // Give up. + device_mapping.push_back(DeviceSpec{InputSourceType::SILENCE, 0}); +found_capture_card: + break; + } + case DeviceSpecProto::ALSA_INPUT: { + // For ALSA, we don't really care about index, but we can use address + // in its place. + + // First see if there's a card that matches on name, num_channels and address. + for (const DeviceSpec &spec : remaining_devices) { + assert(devices.count(spec)); + const DeviceInfo &dev = devices.find(spec)->second; + if (spec.type == InputSourceType::ALSA_INPUT && + dev.alsa_name == device_proto.alsa_name() && + dev.alsa_info == device_proto.alsa_info() && + int(dev.num_channels) == device_proto.num_channels() && + dev.alsa_address == device_proto.address()) { + device_mapping.push_back(spec); + remaining_devices.erase(spec); + goto found_alsa_input; + } + } + + // Looser check: Ignore the address. + for (const DeviceSpec &spec : remaining_devices) { + assert(devices.count(spec)); + const DeviceInfo &dev = devices.find(spec)->second; + if (spec.type == InputSourceType::ALSA_INPUT && + dev.alsa_name == device_proto.alsa_name() && + dev.alsa_info == device_proto.alsa_info() && + int(dev.num_channels) == device_proto.num_channels()) { + device_mapping.push_back(spec); + remaining_devices.erase(spec); + goto found_alsa_input; + } + } + + // OK, so we couldn't map this to a device, but perhaps one is added + // at some point in the future through hotplug. Create a dead card + // matching this one; right now, it will give only silence, + // but it could be replaced with something later. + // + // NOTE: There's a potential race condition here, if the card + // gets inserted while we're doing the device remapping + // (or perhaps more realistically, while we're reading the + // input mapping from disk). + DeviceSpec dead_card_spec; + dead_card_spec = global_audio_mixer->create_dead_card( + device_proto.alsa_name(), device_proto.alsa_info(), device_proto.num_channels()); + device_mapping.push_back(dead_card_spec); + +found_alsa_input: + break; + } + default: + assert(false); + } + } + + for (const BusProto &bus_proto : mapping_proto.bus()) { + if (bus_proto.device_index() < 0 || unsigned(bus_proto.device_index()) >= device_mapping.size()) { + return false; + } + InputMapping::Bus bus; + bus.name = bus_proto.name(); + bus.device = device_mapping[bus_proto.device_index()]; + bus.source_channel[0] = bus_proto.source_channel_left(); + bus.source_channel[1] = bus_proto.source_channel_right(); + new_mapping->buses.push_back(bus); + } + + return true; +} diff --git a/nageru/input_mapping.h b/nageru/input_mapping.h new file mode 100644 index 0000000..67af0f4 --- /dev/null +++ b/nageru/input_mapping.h @@ -0,0 +1,61 @@ +#ifndef _INPUT_MAPPING_H +#define _INPUT_MAPPING_H 1 + +#include +#include +#include +#include + +enum class InputSourceType { SILENCE, CAPTURE_CARD, ALSA_INPUT, FFMPEG_VIDEO_INPUT }; +struct DeviceSpec { + InputSourceType type; + unsigned index; + + bool operator== (const DeviceSpec &other) const { + return type == other.type && index == other.index; + } + + bool operator< (const DeviceSpec &other) const { + if (type != other.type) + return type < other.type; + return index < other.index; + } +}; +struct DeviceInfo { + std::string display_name; + unsigned num_channels; + std::string alsa_name, alsa_info, alsa_address; // ALSA devices only, obviously. +}; + +static inline uint64_t DeviceSpec_to_key(const DeviceSpec &device_spec) +{ + return (uint64_t(device_spec.type) << 32) | device_spec.index; +} + +static inline DeviceSpec key_to_DeviceSpec(uint64_t key) +{ + return DeviceSpec{ InputSourceType(key >> 32), unsigned(key & 0xffffffff) }; +} + +struct InputMapping { + struct Bus { + std::string name; + DeviceSpec device; + int source_channel[2] { -1, -1 }; // Left and right. -1 = none. + }; + + std::vector buses; +}; + +// This is perhaps not the most user-friendly output, but it's at least better +// than the raw index. +std::string spec_to_string(DeviceSpec device_spec); + +bool save_input_mapping_to_file(const std::map &devices, + const InputMapping &mapping, + const std::string &filename); +bool load_input_mapping_from_file(const std::map &devices, + const std::string &filename, + InputMapping *mapping); + +#endif // !defined(_INPUT_MAPPING_H) diff --git a/nageru/input_mapping.ui b/nageru/input_mapping.ui new file mode 100644 index 0000000..4487b94 --- /dev/null +++ b/nageru/input_mapping.ui @@ -0,0 +1,179 @@ + + + InputMappingDialog + + + + 0 + 0 + 879 + 583 + + + + Input mapping + + + + + + + + + + + + Device + + + + + Left input + + + + + Right input + + + + + + + + + + + 0 + 0 + + + + + 30 + 30 + + + + + + + + .. + + + + + + + + 0 + 0 + + + + + 30 + 30 + + + + + + + + .. + + + + + + + + 30 + 30 + + + + + + + + .. + + + + + + + + 30 + 30 + + + + + + + + .. + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + &Save… + + + + + + + &Load… + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + + + + diff --git a/nageru/input_mapping_dialog.cpp b/nageru/input_mapping_dialog.cpp new file mode 100644 index 0000000..23e8895 --- /dev/null +++ b/nageru/input_mapping_dialog.cpp @@ -0,0 +1,326 @@ +#include "input_mapping_dialog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "alsa_pool.h" +#include "defs.h" +#include "post_to_main_thread.h" +#include "ui_input_mapping.h" + +using namespace std; +using namespace std::placeholders; + +InputMappingDialog::InputMappingDialog() + : ui(new Ui::InputMappingDialog), + mapping(global_audio_mixer->get_input_mapping()), + old_mapping(mapping), + devices(global_audio_mixer->get_devices()) +{ + for (unsigned bus_index = 0; bus_index < mapping.buses.size(); ++bus_index) { + bus_settings.push_back(global_audio_mixer->get_bus_settings(bus_index)); + } + + ui->setupUi(this); + ui->table->setSelectionBehavior(QAbstractItemView::SelectRows); + ui->table->setSelectionMode(QAbstractItemView::SingleSelection); // Makes implementing moving easier for now. + + fill_ui_from_mapping(mapping); + connect(ui->table, &QTableWidget::cellChanged, this, &InputMappingDialog::cell_changed); + connect(ui->ok_cancel_buttons, &QDialogButtonBox::accepted, this, &InputMappingDialog::ok_clicked); + connect(ui->ok_cancel_buttons, &QDialogButtonBox::rejected, this, &InputMappingDialog::cancel_clicked); + connect(ui->add_button, &QPushButton::clicked, this, &InputMappingDialog::add_clicked); + connect(ui->remove_button, &QPushButton::clicked, this, &InputMappingDialog::remove_clicked); + connect(ui->up_button, &QPushButton::clicked, bind(&InputMappingDialog::updown_clicked, this, -1)); + connect(ui->down_button, &QPushButton::clicked, bind(&InputMappingDialog::updown_clicked, this, 1)); + connect(ui->save_button, &QPushButton::clicked, this, &InputMappingDialog::save_clicked); + connect(ui->load_button, &QPushButton::clicked, this, &InputMappingDialog::load_clicked); + + update_button_state(); + connect(ui->table, &QTableWidget::itemSelectionChanged, this, &InputMappingDialog::update_button_state); + + saved_callback = global_audio_mixer->get_state_changed_callback(); + global_audio_mixer->set_state_changed_callback([this]{ + post_to_main_thread([this]{ + devices = global_audio_mixer->get_devices(); + for (unsigned row = 0; row < mapping.buses.size(); ++row) { + fill_row_from_bus(row, mapping.buses[row]); + } + }); + }); +} + +InputMappingDialog::~InputMappingDialog() +{ + global_audio_mixer->set_state_changed_callback(saved_callback); +} + +void InputMappingDialog::fill_ui_from_mapping(const InputMapping &mapping) +{ + ui->table->verticalHeader()->hide(); + ui->table->horizontalHeader()->setSectionResizeMode(1, QHeaderView::ResizeToContents); + ui->table->horizontalHeader()->setSectionResizeMode(2, QHeaderView::ResizeToContents); + ui->table->horizontalHeader()->setSectionResizeMode(3, QHeaderView::ResizeToContents); + ui->table->horizontalHeader()->setSectionsClickable(false); + + ui->table->setRowCount(mapping.buses.size()); + for (unsigned row = 0; row < mapping.buses.size(); ++row) { + fill_row_from_bus(row, mapping.buses[row]); + } +} + +void InputMappingDialog::fill_row_from_bus(unsigned row, const InputMapping::Bus &bus) +{ + QString name(QString::fromStdString(bus.name)); + ui->table->setItem(row, 0, new QTableWidgetItem(name)); + + // Card choices. If there's already a combobox here, we try to modify + // the elements in-place, so that the UI doesn't go away under the user's feet + // if they are in the process of choosing an item. + QComboBox *card_combo = static_cast(ui->table->cellWidget(row, 1)); + if (card_combo == nullptr) { + card_combo = new QComboBox; + } + unsigned current_index = 0; + if (card_combo->count() == 0) { + card_combo->addItem(QString("(none) ")); + } + for (const auto &spec_and_info : devices) { + QString label(QString::fromStdString(spec_and_info.second.display_name)); + if (spec_and_info.first.type == InputSourceType::ALSA_INPUT) { + ALSAPool::Device::State state = global_audio_mixer->get_alsa_card_state(spec_and_info.first.index); + if (state == ALSAPool::Device::State::EMPTY) { + continue; + } else if (state == ALSAPool::Device::State::STARTING) { + label += " (busy)"; + } else if (state == ALSAPool::Device::State::DEAD) { + label += " (dead)"; + } + } + ++current_index; + if (unsigned(card_combo->count()) > current_index) { + card_combo->setItemText(current_index, label + " "); + card_combo->setItemData(current_index, qulonglong(DeviceSpec_to_key(spec_and_info.first))); + } else { + card_combo->addItem( + label + " ", + qulonglong(DeviceSpec_to_key(spec_and_info.first))); + } + if (bus.device == spec_and_info.first) { + card_combo->setCurrentIndex(current_index); + } + } + // Remove any excess items from earlier. (This is only for paranoia; + // they should be held, so it shouldn't matter.) + while (unsigned(card_combo->count()) > current_index + 1) { + card_combo->removeItem(current_index + 1); + } + connect(card_combo, static_cast(&QComboBox::currentIndexChanged), + bind(&InputMappingDialog::card_selected, this, card_combo, row, _1)); + ui->table->setCellWidget(row, 1, card_combo); + + setup_channel_choices_from_bus(row, bus); +} + +void InputMappingDialog::setup_channel_choices_from_bus(unsigned row, const InputMapping::Bus &bus) +{ + // Left and right channel. + // TODO: If there's already a widget here, modify it instead of creating a new one, + // as we do with card choices. + for (unsigned channel = 0; channel < 2; ++channel) { + QComboBox *channel_combo = new QComboBox; + channel_combo->addItem(QString("(none)")); + if (bus.device.type == InputSourceType::CAPTURE_CARD || + bus.device.type == InputSourceType::ALSA_INPUT || + bus.device.type == InputSourceType::FFMPEG_VIDEO_INPUT) { + auto device_it = devices.find(bus.device); + assert(device_it != devices.end()); + unsigned num_device_channels = device_it->second.num_channels; + for (unsigned source = 0; source < num_device_channels; ++source) { + char buf[256]; + snprintf(buf, sizeof(buf), "Channel %u ", source + 1); + channel_combo->addItem(QString(buf)); + } + channel_combo->setCurrentIndex(bus.source_channel[channel] + 1); + } else { + assert(bus.device.type == InputSourceType::SILENCE); + channel_combo->setCurrentIndex(0); + } + connect(channel_combo, static_cast(&QComboBox::currentIndexChanged), + bind(&InputMappingDialog::channel_selected, this, row, channel, _1)); + ui->table->setCellWidget(row, 2 + channel, channel_combo); + } +} + +void InputMappingDialog::ok_clicked() +{ + global_audio_mixer->set_state_changed_callback(saved_callback); + global_audio_mixer->set_input_mapping(mapping); + for (unsigned bus_index = 0; bus_index < mapping.buses.size(); ++bus_index) { + global_audio_mixer->set_bus_settings(bus_index, bus_settings[bus_index]); + global_audio_mixer->reset_peak(bus_index); + } + accept(); +} + +void InputMappingDialog::cancel_clicked() +{ + global_audio_mixer->set_state_changed_callback(saved_callback); + global_audio_mixer->set_input_mapping(old_mapping); + reject(); +} + +void InputMappingDialog::cell_changed(int row, int column) +{ + if (column != 0) { + // Spurious; only really the name column should fire these. + return; + } + mapping.buses[row].name = ui->table->item(row, column)->text().toStdString(); +} + +void InputMappingDialog::card_selected(QComboBox *card_combo, unsigned row, int index) +{ + uint64_t key = card_combo->itemData(index).toULongLong(); + mapping.buses[row].device = key_to_DeviceSpec(key); + setup_channel_choices_from_bus(row, mapping.buses[row]); +} + +void InputMappingDialog::channel_selected(unsigned row, unsigned channel, int index) +{ + mapping.buses[row].source_channel[channel] = index - 1; +} + +void InputMappingDialog::add_clicked() +{ + QTableWidgetSelectionRange all(0, 0, ui->table->rowCount() - 1, ui->table->columnCount() - 1); + ui->table->setRangeSelected(all, false); + + InputMapping::Bus new_bus; + new_bus.name = "New input"; + new_bus.device.type = InputSourceType::SILENCE; + mapping.buses.push_back(new_bus); + bus_settings.push_back(AudioMixer::get_default_bus_settings()); + ui->table->setRowCount(mapping.buses.size()); + + unsigned row = mapping.buses.size() - 1; + fill_row_from_bus(row, new_bus); + ui->table->editItem(ui->table->item(row, 0)); // Start editing the name. + update_button_state(); +} + +void InputMappingDialog::remove_clicked() +{ + assert(ui->table->rowCount() != 0); + + set> rows_to_delete; // Need to remove in reverse order. + for (const QTableWidgetSelectionRange &range : ui->table->selectedRanges()) { + for (int row = range.topRow(); row <= range.bottomRow(); ++row) { + rows_to_delete.insert(row); + } + } + if (rows_to_delete.empty()) { + rows_to_delete.insert(ui->table->rowCount() - 1); + } + + for (int row : rows_to_delete) { + ui->table->removeRow(row); + mapping.buses.erase(mapping.buses.begin() + row); + bus_settings.erase(bus_settings.begin() + row); + } + update_button_state(); +} + +void InputMappingDialog::updown_clicked(int direction) +{ + assert(ui->table->selectedRanges().size() == 1); + const QTableWidgetSelectionRange &range = ui->table->selectedRanges()[0]; + int a_row = range.bottomRow(); + int b_row = range.bottomRow() + direction; + + swap(mapping.buses[a_row], mapping.buses[b_row]); + swap(bus_settings[a_row], bus_settings[b_row]); + fill_row_from_bus(a_row, mapping.buses[a_row]); + fill_row_from_bus(b_row, mapping.buses[b_row]); + + QTableWidgetSelectionRange a_sel(a_row, 0, a_row, ui->table->columnCount() - 1); + QTableWidgetSelectionRange b_sel(b_row, 0, b_row, ui->table->columnCount() - 1); + ui->table->setRangeSelected(a_sel, false); + ui->table->setRangeSelected(b_sel, true); +} + +void InputMappingDialog::save_clicked() +{ +#if HAVE_CEF + // The native file dialog uses GTK+, which interferes with CEF's use of the GLib main loop. + QFileDialog::Option options(QFileDialog::DontUseNativeDialog); +#else + QFileDialog::Option options(QFileDialog::Option(0)); +#endif + QString filename = QFileDialog::getSaveFileName(this, + "Save input mapping", QString(), tr("Mapping files (*.mapping)"), /*selectedFilter=*/nullptr, options); + if (!filename.endsWith(".mapping")) { + filename += ".mapping"; + } + if (!save_input_mapping_to_file(devices, mapping, filename.toStdString())) { + QMessageBox box; + box.setText("Could not save mapping to '" + filename + "'. Check that you have the right permissions and try again."); + box.exec(); + } +} + +void InputMappingDialog::load_clicked() +{ +#if HAVE_CEF + // The native file dialog uses GTK+, which interferes with CEF's use of the GLib main loop. + QFileDialog::Option options(QFileDialog::DontUseNativeDialog); +#else + QFileDialog::Option options(QFileDialog::Option(0)); +#endif + QString filename = QFileDialog::getOpenFileName(this, + "Load input mapping", QString(), tr("Mapping files (*.mapping)"), /*selectedFilter=*/nullptr, options); + InputMapping new_mapping; + if (!load_input_mapping_from_file(devices, filename.toStdString(), &new_mapping)) { + QMessageBox box; + box.setText("Could not load mapping from '" + filename + "'. Check that the file exists, has the right permissions and is valid."); + box.exec(); + return; + } + + mapping = new_mapping; + bus_settings.clear(); + for (unsigned bus_index = 0; bus_index < mapping.buses.size(); ++bus_index) { + bus_settings.push_back(global_audio_mixer->get_bus_settings(bus_index)); + } + devices = global_audio_mixer->get_devices(); // New dead cards may have been made. + fill_ui_from_mapping(mapping); +} + +void InputMappingDialog::update_button_state() +{ + ui->add_button->setDisabled(mapping.buses.size() >= MAX_BUSES); + ui->remove_button->setDisabled(mapping.buses.size() == 0); + ui->up_button->setDisabled( + ui->table->selectedRanges().empty() || + ui->table->selectedRanges()[0].bottomRow() == 0); + ui->down_button->setDisabled( + ui->table->selectedRanges().empty() || + ui->table->selectedRanges()[0].bottomRow() == ui->table->rowCount() - 1); +} diff --git a/nageru/input_mapping_dialog.h b/nageru/input_mapping_dialog.h new file mode 100644 index 0000000..640644e --- /dev/null +++ b/nageru/input_mapping_dialog.h @@ -0,0 +1,61 @@ +#ifndef _INPUT_MAPPING_DIALOG_H +#define _INPUT_MAPPING_DIALOG_H + +#include +#include +#include +#include + +#include "audio_mixer.h" +#include "input_mapping.h" + +class QObject; + +namespace Ui { +class InputMappingDialog; +} // namespace Ui + +class QComboBox; + +class InputMappingDialog : public QDialog +{ + Q_OBJECT + +public: + InputMappingDialog(); + ~InputMappingDialog(); + +private: + void fill_ui_from_mapping(const InputMapping &mapping); + void fill_row_from_bus(unsigned row, const InputMapping::Bus &bus); + void setup_channel_choices_from_bus(unsigned row, const InputMapping::Bus &bus); + void cell_changed(int row, int column); + void card_selected(QComboBox *card_combo, unsigned row, int index); + void channel_selected(unsigned row, unsigned channel, int index); + void ok_clicked(); + void cancel_clicked(); + void add_clicked(); + void remove_clicked(); + void updown_clicked(int direction); + void save_clicked(); + void load_clicked(); + void update_button_state(); + + Ui::InputMappingDialog *ui; + InputMapping mapping; // Under edit. Will be committed on OK. + + // The old mapping. Will be re-committed on cancel, so that we + // unhold all the unused devices (otherwise they would be + // held forever). + InputMapping old_mapping; + + // One for each bus in the mapping. Edited along with the mapping, + // so that old volumes etc. are being kept in place for buses that + // existed before. + std::vector bus_settings; + + std::map devices; // Needs no lock, accessed only on the UI thread. + AudioMixer::state_changed_callback_t saved_callback; +}; + +#endif // !defined(_INPUT_MAPPING_DIALOG_H) diff --git a/nageru/input_state.h b/nageru/input_state.h new file mode 100644 index 0000000..2f33654 --- /dev/null +++ b/nageru/input_state.h @@ -0,0 +1,34 @@ +#ifndef _INPUT_STATE_H +#define _INPUT_STATE_H 1 + +#include + +#include "defs.h" +#include "ref_counted_frame.h" + +struct BufferedFrame { + RefCountedFrame frame; + unsigned field_number; +}; + +// Encapsulates the state of all inputs at any given instant. +// In particular, this is captured by Theme::get_chain(), +// so that it can hold on to all the frames it needs for rendering. +struct InputState { + // For each card, the last five frames (or fields), with 0 being the + // most recent one. Note that we only need the actual history if we have + // interlaced output (for deinterlacing), so if we detect progressive input, + // we immediately clear out all history and all entries will point to the same + // frame. + BufferedFrame buffered_frames[MAX_VIDEO_CARDS][FRAME_HISTORY_LENGTH]; + + // For each card, the current Y'CbCr input settings. Ignored for BGRA inputs. + // If ycbcr_coefficients_auto = true for a given card, the others are ignored + // for that card (SD is taken to be Rec. 601, HD is taken to be Rec. 709, + // both limited range). + bool ycbcr_coefficients_auto[MAX_VIDEO_CARDS]; + movit::YCbCrLumaCoefficients ycbcr_coefficients[MAX_VIDEO_CARDS]; + bool full_range[MAX_VIDEO_CARDS]; +}; + +#endif // !defined(_INPUT_STATE_H) diff --git a/nageru/json.proto b/nageru/json.proto new file mode 100644 index 0000000..55e642a --- /dev/null +++ b/nageru/json.proto @@ -0,0 +1,14 @@ +// Messages used to produce JSON (it's the simplest way we can create valid +// JSON without pulling in an external JSON library). + +syntax = "proto2"; + +message Channels { + repeated Channel channel = 1; +} + +message Channel { + required int32 index = 1; + required string name = 2; + required string color = 3; +} diff --git a/nageru/kaeru.cpp b/nageru/kaeru.cpp new file mode 100644 index 0000000..10f1e93 --- /dev/null +++ b/nageru/kaeru.cpp @@ -0,0 +1,225 @@ +// Kaeru (換える), a simple transcoder intended for use with Nageru. + +#include "audio_encoder.h" +#include "basic_stats.h" +#include "defs.h" +#include "flags.h" +#include "ffmpeg_capture.h" +#include "mixer.h" +#include "mux.h" +#include "quittable_sleeper.h" +#include "timebase.h" +#include "x264_encoder.h" + +#include +#include +#include +#include +#include +#include + +using namespace bmusb; +using namespace movit; +using namespace std; +using namespace std::chrono; +using namespace std::placeholders; + +Mixer *global_mixer = nullptr; +X264Encoder *global_x264_encoder = nullptr; +int frame_num = 0; +BasicStats *global_basic_stats = nullptr; +QuittableSleeper should_quit; +MuxMetrics stream_mux_metrics; + +int write_packet(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time) +{ + static bool seen_sync_markers = false; + static string stream_mux_header; + HTTPD *httpd = (HTTPD *)opaque; + + if (type == AVIO_DATA_MARKER_SYNC_POINT || type == AVIO_DATA_MARKER_BOUNDARY_POINT) { + seen_sync_markers = true; + } else if (type == AVIO_DATA_MARKER_UNKNOWN && !seen_sync_markers) { + // We don't know if this is a keyframe or not (the muxer could + // avoid marking it), so we just have to make the best of it. + type = AVIO_DATA_MARKER_SYNC_POINT; + } + + if (type == AVIO_DATA_MARKER_HEADER) { + stream_mux_header.append((char *)buf, buf_size); + httpd->set_header(stream_mux_header); + } else { + httpd->add_data((char *)buf, buf_size, type == AVIO_DATA_MARKER_SYNC_POINT, time, AVRational{ AV_TIME_BASE, 1 }); + } + return buf_size; +} + +unique_ptr create_mux(HTTPD *httpd, AVOutputFormat *oformat, X264Encoder *x264_encoder, AudioEncoder *audio_encoder) +{ + AVFormatContext *avctx = avformat_alloc_context(); + avctx->oformat = oformat; + + uint8_t *buf = (uint8_t *)av_malloc(MUX_BUFFER_SIZE); + avctx->pb = avio_alloc_context(buf, MUX_BUFFER_SIZE, 1, httpd, nullptr, nullptr, nullptr); + avctx->pb->write_data_type = &write_packet; + avctx->pb->ignore_boundary_point = 1; + avctx->flags = AVFMT_FLAG_CUSTOM_IO; + + string video_extradata = x264_encoder->get_global_headers(); + + unique_ptr mux; + mux.reset(new Mux(avctx, global_flags.width, global_flags.height, Mux::CODEC_H264, video_extradata, audio_encoder->get_codec_parameters().get(), COARSE_TIMEBASE, + /*write_callback=*/nullptr, Mux::WRITE_FOREGROUND, { &stream_mux_metrics })); + stream_mux_metrics.init({{ "destination", "http" }}); + return mux; +} + +void video_frame_callback(FFmpegCapture *video, X264Encoder *x264_encoder, AudioEncoder *audio_encoder, + int64_t video_pts, AVRational video_timebase, + int64_t audio_pts, AVRational audio_timebase, + uint16_t timecode, + FrameAllocator::Frame video_frame, size_t video_offset, VideoFormat video_format, + FrameAllocator::Frame audio_frame, size_t audio_offset, AudioFormat audio_format) +{ + if (video_pts >= 0 && video_frame.len > 0) { + ReceivedTimestamps ts; + ts.ts.push_back(steady_clock::now()); + + video_pts = av_rescale_q(video_pts, video_timebase, AVRational{ 1, TIMEBASE }); + int64_t frame_duration = TIMEBASE * video_format.frame_rate_den / video_format.frame_rate_nom; + x264_encoder->add_frame(video_pts, frame_duration, video->get_current_frame_ycbcr_format().luma_coefficients, video_frame.data + video_offset, ts); + global_basic_stats->update(frame_num++, /*dropped_frames=*/0); + } + if (audio_frame.len > 0) { + // FFmpegCapture takes care of this for us. + assert(audio_format.num_channels == 2); + assert(audio_format.sample_rate == OUTPUT_FREQUENCY); + + // TODO: Reduce some duplication against AudioMixer here. + size_t num_samples = audio_frame.len / (audio_format.bits_per_sample / 8); + vector float_samples; + float_samples.resize(num_samples); + if (audio_format.bits_per_sample == 16) { + const int16_t *src = (const int16_t *)audio_frame.data; + float *dst = &float_samples[0]; + for (size_t i = 0; i < num_samples; ++i) { + *dst++ = le16toh(*src++) * (1.0f / 32768.0f); + } + } else if (audio_format.bits_per_sample == 32) { + const int32_t *src = (const int32_t *)audio_frame.data; + float *dst = &float_samples[0]; + for (size_t i = 0; i < num_samples; ++i) { + *dst++ = le32toh(*src++) * (1.0f / 2147483648.0f); + } + } else { + assert(false); + } + audio_pts = av_rescale_q(audio_pts, audio_timebase, AVRational{ 1, TIMEBASE }); + audio_encoder->encode_audio(float_samples, audio_pts); + } + + if (video_frame.owner) { + video_frame.owner->release_frame(video_frame); + } + if (audio_frame.owner) { + audio_frame.owner->release_frame(audio_frame); + } +} + +void audio_frame_callback(Mux *mux, const AVPacket *pkt, AVRational timebase) +{ + mux->add_packet(*pkt, pkt->pts, pkt->dts == AV_NOPTS_VALUE ? pkt->pts : pkt->dts, timebase, /*stream_index=*/1); +} + +void adjust_bitrate(int signal) +{ + int new_bitrate = global_flags.x264_bitrate; + if (signal == SIGUSR1) { + new_bitrate += 100; + if (new_bitrate > 100000) { + fprintf(stderr, "Ignoring SIGUSR1, can't increase bitrate below 100000 kbit/sec (currently at %d kbit/sec)\n", + global_flags.x264_bitrate); + } else { + fprintf(stderr, "Increasing bitrate to %d kbit/sec due to SIGUSR1.\n", new_bitrate); + global_flags.x264_bitrate = new_bitrate; + global_x264_encoder->change_bitrate(new_bitrate); + } + } else if (signal == SIGUSR2) { + new_bitrate -= 100; + if (new_bitrate < 100) { + fprintf(stderr, "Ignoring SIGUSR2, can't decrease bitrate below 100 kbit/sec (currently at %d kbit/sec)\n", + global_flags.x264_bitrate); + } else { + fprintf(stderr, "Decreasing bitrate to %d kbit/sec due to SIGUSR2.\n", new_bitrate); + global_flags.x264_bitrate = new_bitrate; + global_x264_encoder->change_bitrate(new_bitrate); + } + } +} + +void request_quit(int signal) +{ + should_quit.quit(); +} + +int main(int argc, char *argv[]) +{ + parse_flags(PROGRAM_KAERU, argc, argv); + if (optind + 1 != argc) { + usage(PROGRAM_KAERU); + exit(1); + } + global_flags.num_cards = 1; // For latency metrics. + +#if LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(58, 9, 100) + av_register_all(); +#endif + avformat_network_init(); + + HTTPD httpd; + + AVOutputFormat *oformat = av_guess_format(global_flags.stream_mux_name.c_str(), nullptr, nullptr); + assert(oformat != nullptr); + + unique_ptr audio_encoder; + if (global_flags.stream_audio_codec_name.empty()) { + audio_encoder.reset(new AudioEncoder(AUDIO_OUTPUT_CODEC_NAME, DEFAULT_AUDIO_OUTPUT_BIT_RATE, oformat)); + } else { + audio_encoder.reset(new AudioEncoder(global_flags.stream_audio_codec_name, global_flags.stream_audio_codec_bitrate, oformat)); + } + + unique_ptr x264_encoder(new X264Encoder(oformat)); + unique_ptr http_mux = create_mux(&httpd, oformat, x264_encoder.get(), audio_encoder.get()); + if (global_flags.transcode_audio) { + audio_encoder->add_mux(http_mux.get()); + } + x264_encoder->add_mux(http_mux.get()); + global_x264_encoder = x264_encoder.get(); + + FFmpegCapture video(argv[optind], global_flags.width, global_flags.height); + video.set_pixel_format(FFmpegCapture::PixelFormat_NV12); + video.set_frame_callback(bind(video_frame_callback, &video, x264_encoder.get(), audio_encoder.get(), _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11)); + if (!global_flags.transcode_audio) { + video.set_audio_callback(bind(audio_frame_callback, http_mux.get(), _1, _2)); + } + video.configure_card(); + video.start_bm_capture(); + video.change_rate(2.0); // Be sure never to really fall behind, but also don't dump huge amounts of stuff onto x264. + + BasicStats basic_stats(/*verbose=*/false, /*use_opengl=*/false); + global_basic_stats = &basic_stats; + httpd.start(global_flags.http_port); + + signal(SIGUSR1, adjust_bitrate); + signal(SIGUSR2, adjust_bitrate); + signal(SIGINT, request_quit); + + while (!should_quit.should_quit()) { + should_quit.sleep_for(hours(1000)); + } + + video.stop_dequeue_thread(); + // Stop the x264 encoder before killing the mux it's writing to. + x264_encoder.reset(); + return 0; +} diff --git a/nageru/lrameter.cpp b/nageru/lrameter.cpp new file mode 100644 index 0000000..62b4a9f --- /dev/null +++ b/nageru/lrameter.cpp @@ -0,0 +1,111 @@ +#include "lrameter.h" + +#include +#include +#include +#include + +#include "vu_common.h" + +class QPaintEvent; +class QResizeEvent; + +using namespace std; + +LRAMeter::LRAMeter(QWidget *parent) + : QWidget(parent) +{ +} + +void LRAMeter::resizeEvent(QResizeEvent *event) +{ + recalculate_pixmaps(); +} + +void LRAMeter::paintEvent(QPaintEvent *event) +{ + QPainter painter(this); + + float level_lufs; + float range_low_lufs; + float range_high_lufs; + { + unique_lock lock(level_mutex); + level_lufs = this->level_lufs; + range_low_lufs = this->range_low_lufs; + range_high_lufs = this->range_high_lufs; + } + + float level_lu = level_lufs - ref_level_lufs; + float range_low_lu = range_low_lufs - ref_level_lufs; + float range_high_lu = range_high_lufs - ref_level_lufs; + int range_low_pos = lrint(lufs_to_pos(range_low_lu, height())); + int range_high_pos = lrint(lufs_to_pos(range_high_lu, height())); + + QRect top_off_rect(0, 0, width(), range_high_pos); + QRect on_rect(0, range_high_pos, width(), range_low_pos - range_high_pos); + QRect bottom_off_rect(0, range_low_pos, width(), height() - range_low_pos); + + painter.drawPixmap(top_off_rect, off_pixmap, top_off_rect); + painter.drawPixmap(on_rect, on_pixmap, on_rect); + painter.drawPixmap(bottom_off_rect, off_pixmap, bottom_off_rect); + + // Draw the target area (+/-1 LU is allowed EBU range). + // It turns green when we're within. + int min_y = lrint(lufs_to_pos(1.0f, height())); + int max_y = lrint(lufs_to_pos(-1.0f, height())); + + // FIXME: This outlining isn't so pretty. + { + QPen pen(Qt::black); + pen.setWidth(5); + painter.setPen(pen); + painter.drawRect(2, min_y, width() - 5, max_y - min_y); + } + { + QPen pen; + if (level_lu >= -1.0f && level_lu <= 1.0f) { + pen.setColor(Qt::green); + } else { + pen.setColor(Qt::red); + } + pen.setWidth(3); + painter.setPen(pen); + painter.drawRect(2, min_y, width() - 5, max_y - min_y); + } + + // Draw the integrated loudness meter, in the same color as the target area. + int y = lrint(lufs_to_pos(level_lu, height())); + { + QPen pen(Qt::black); + pen.setWidth(5); + painter.setPen(pen); + painter.drawRect(2, y, width() - 5, 1); + } + { + QPen pen; + if (level_lu >= -1.0f && level_lu <= 1.0f) { + pen.setColor(Qt::green); + } else { + pen.setColor(Qt::red); + } + pen.setWidth(3); + painter.setPen(pen); + painter.drawRect(2, y, width() - 5, 1); + } +} + +void LRAMeter::recalculate_pixmaps() +{ + const int margin = 5; + + on_pixmap = QPixmap(width(), height()); + QPainter on_painter(&on_pixmap); + on_painter.fillRect(0, 0, width(), height(), parentWidget()->palette().window()); + draw_vu_meter(on_painter, width(), height(), margin, 2.0, true, min_level, max_level, /*flip=*/false); + + off_pixmap = QPixmap(width(), height()); + QPainter off_painter(&off_pixmap); + off_painter.fillRect(0, 0, width(), height(), parentWidget()->palette().window()); + draw_vu_meter(off_painter, width(), height(), margin, 2.0, false, min_level, max_level, /*flip=*/false); +} diff --git a/nageru/lrameter.h b/nageru/lrameter.h new file mode 100644 index 0000000..7a832df --- /dev/null +++ b/nageru/lrameter.h @@ -0,0 +1,67 @@ +#ifndef LRAMETER_H +#define LRAMETER_H + +#include +#include +#include +#include +#include + +#include "vu_common.h" + +class QObject; +class QPaintEvent; +class QResizeEvent; + +class LRAMeter : public QWidget +{ + Q_OBJECT + +public: + LRAMeter(QWidget *parent); + + void set_levels(float level_lufs, float range_low_lufs, float range_high_lufs) { + std::unique_lock lock(level_mutex); + this->level_lufs = level_lufs; + this->range_low_lufs = range_low_lufs; + this->range_high_lufs = range_high_lufs; + QMetaObject::invokeMethod(this, "update", Qt::AutoConnection); + } + + double lufs_to_pos(float level_lu, int height) + { + return ::lufs_to_pos(level_lu, height, min_level, max_level); + } + + void set_min_level(float min_level) + { + this->min_level = min_level; + recalculate_pixmaps(); + } + + void set_max_level(float max_level) + { + this->max_level = max_level; + recalculate_pixmaps(); + } + + void set_ref_level(float ref_level_lufs) + { + this->ref_level_lufs = ref_level_lufs; + } + +private: + void resizeEvent(QResizeEvent *event) override; + void paintEvent(QPaintEvent *event) override; + void recalculate_pixmaps(); + + std::mutex level_mutex; + float level_lufs = -HUGE_VAL; + float range_low_lufs = -HUGE_VAL; + float range_high_lufs = -HUGE_VAL; + float min_level = -18.0f, max_level = 9.0f, ref_level_lufs = -23.0f; + + QPixmap on_pixmap, off_pixmap; +}; + +#endif diff --git a/nageru/main.cpp b/nageru/main.cpp new file mode 100644 index 0000000..c1a52c0 --- /dev/null +++ b/nageru/main.cpp @@ -0,0 +1,128 @@ +extern "C" { +#include +} +#include +#include +#include +#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_CEF +#include +#include +#include +#include +#endif + +#include "basic_stats.h" +#ifdef HAVE_CEF +#include "nageru_cef_app.h" +#endif +#include "context.h" +#include "flags.h" +#include "image_input.h" +#include "mainwindow.h" +#include "mixer.h" +#include "quicksync_encoder.h" + +#ifdef HAVE_CEF +CefRefPtr cef_app; +#endif + +int main(int argc, char *argv[]) +{ +#ifdef HAVE_CEF + // Let CEF have first priority on parsing the command line, because we might be + // launched as a CEF sub-process. + CefMainArgs main_args(argc, argv); + cef_app = CefRefPtr(new NageruCefApp()); + int err = CefExecuteProcess(main_args, cef_app.get(), nullptr); + if (err >= 0) { + return err; + } + + // CEF wants to use GLib for its main loop, which interferes with Qt's use of it. + // The alternative is trying to integrate CEF into Qt's main loop, but that requires + // fairly extensive cross-thread communication and that parts of CEF runs on Qt's UI + // thread. + setenv("QT_NO_GLIB", "1", 0); +#endif + + parse_flags(PROGRAM_NAGERU, argc, argv); + + if (global_flags.va_display.empty() && !global_flags.x264_video_to_disk) { + // The user didn't specify a VA-API display, but we need one. + // See if the default works, and if not, let's try to help + // the user by seeing if there's any that would work automatically. + global_flags.va_display = QuickSyncEncoder::get_usable_va_display(); + } + + if ((global_flags.va_display.empty() || + global_flags.va_display[0] != '/') && !global_flags.x264_video_to_disk) { + // We normally use EGL for zerocopy, but if we use VA against DRM + // instead of against X11, we turn it off, and then don't need EGL. + setenv("QT_XCB_GL_INTEGRATION", "xcb_egl", 0); + using_egl = true; + } + setlinebuf(stdout); +#if LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(58, 9, 100) + av_register_all(); +#endif + + QCoreApplication::setAttribute(Qt::AA_ShareOpenGLContexts, true); + + QSurfaceFormat fmt; + fmt.setDepthBufferSize(0); + fmt.setStencilBufferSize(0); + fmt.setProfile(QSurfaceFormat::CoreProfile); + fmt.setMajorVersion(3); + fmt.setMinorVersion(1); + + // Turn off vsync, since Qt generally gives us at most frame rate + // (display frequency) / (number of QGLWidgets active). + fmt.setSwapInterval(0); + + QSurfaceFormat::setDefaultFormat(fmt); + + QGLFormat::setDefaultFormat(QGLFormat::fromSurfaceFormat(fmt)); + + QApplication app(argc, argv); + global_share_widget = new QGLWidget(); + if (!global_share_widget->isValid()) { + fprintf(stderr, "Failed to initialize OpenGL. Nageru needs at least OpenGL 3.1 to function properly.\n"); + exit(1); + } + + MainWindow mainWindow; + mainWindow.resize(QSize(1500, 910)); + mainWindow.show(); + + app.installEventFilter(&mainWindow); // For white balance color picking. + + // Even on an otherwise unloaded system, it would seem writing the recording + // to disk (potentially terabytes of data as time goes by) causes Nageru + // to be pushed out of RAM. If we have the right privileges, simply lock us + // into memory for better realtime behavior. + if (mlockall(MCL_CURRENT | MCL_FUTURE) == -1) { + perror("mlockall()"); + fprintf(stderr, "Failed to lock Nageru into RAM. You probably want to\n"); + fprintf(stderr, "increase \"memlock\" for your user in limits.conf\n"); + fprintf(stderr, "for better realtime behavior.\n"); + uses_mlock = false; + } else { + uses_mlock = true; + } + + int rc = app.exec(); + global_mixer->quit(); + mainWindow.mixer_shutting_down(); + delete global_mixer; + ImageInput::shutdown_updaters(); + return rc; +} diff --git a/nageru/mainwindow.cpp b/nageru/mainwindow.cpp new file mode 100644 index 0000000..b542c10 --- /dev/null +++ b/nageru/mainwindow.cpp @@ -0,0 +1,1569 @@ +#include "mainwindow.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aboutdialog.h" +#include "alsa_pool.h" +#include "analyzer.h" +#include "clickable_label.h" +#include "context_menus.h" +#include "correlation_meter.h" +#include "disk_space_estimator.h" +#include "ellipsis_label.h" +#include "flags.h" +#include "glwidget.h" +#include "input_mapping.h" +#include "input_mapping_dialog.h" +#include "lrameter.h" +#include "midi_mapping.pb.h" +#include "midi_mapping_dialog.h" +#include "mixer.h" +#include "nonlinear_fader.h" +#include "post_to_main_thread.h" +#include "ui_audio_expanded_view.h" +#include "ui_audio_miniview.h" +#include "ui_display.h" +#include "ui_mainwindow.h" +#include "vumeter.h" + +using namespace std; +using namespace std::chrono; +using namespace std::placeholders; + +Q_DECLARE_METATYPE(std::string); +Q_DECLARE_METATYPE(std::vector); + +MainWindow *global_mainwindow = nullptr; + +// -0.1 dBFS is EBU peak limit. We use it consistently, even for the bus meters +// (which don't calculate interpolate peak, and in general don't follow EBU recommendations). +constexpr float peak_limit_dbfs = -0.1f; + +namespace { + +void schedule_cut_signal(int ignored) +{ + global_mixer->schedule_cut(); +} + +void quit_signal(int ignored) +{ + global_mainwindow->close(); +} + +void slave_knob(QDial *master, QDial *slave) +{ + QWidget::connect(master, &QDial::valueChanged, [slave](int value){ + slave->blockSignals(true); + slave->setValue(value); + slave->blockSignals(false); + }); + QWidget::connect(slave, &QDial::valueChanged, [master](int value){ + master->setValue(value); + }); +} + +void slave_checkbox(QCheckBox *master, QCheckBox *slave) +{ + QWidget::connect(master, &QCheckBox::stateChanged, [slave](int state){ + slave->blockSignals(true); + slave->setCheckState(Qt::CheckState(state)); + slave->blockSignals(false); + }); + QWidget::connect(slave, &QCheckBox::stateChanged, [master](int state){ + master->setCheckState(Qt::CheckState(state)); + }); +} + +void slave_fader(NonLinearFader *master, NonLinearFader *slave) +{ + QWidget::connect(master, &NonLinearFader::dbValueChanged, [slave](double value) { + slave->blockSignals(true); + slave->setDbValue(value); + slave->blockSignals(false); + }); + QWidget::connect(slave, &NonLinearFader::dbValueChanged, [master](double value){ + master->setDbValue(value); + }); +} + +constexpr unsigned DB_WITH_SIGN = 0x1; +constexpr unsigned DB_BARE = 0x2; + +string format_db(double db, unsigned flags) +{ + string text; + if (flags & DB_WITH_SIGN) { + if (isfinite(db)) { + char buf[256]; + snprintf(buf, sizeof(buf), "%+.1f", db); + text = buf; + } else if (db < 0.0) { + text = "-∞"; + } else { + // Should never happen, really. + text = "+∞"; + } + } else { + if (isfinite(db)) { + char buf[256]; + snprintf(buf, sizeof(buf), "%.1f", db); + text = buf; + } else if (db < 0.0) { + text = "-∞"; + } else { + // Should never happen, really. + text = "∞"; + } + } + if (!(flags & DB_BARE)) { + text += " dB"; + } + return text; +} + +void set_peak_label(QLabel *peak_label, float peak_db) +{ + peak_label->setText(QString::fromStdString(format_db(peak_db, DB_BARE))); + + if (peak_db > peak_limit_dbfs) { + peak_label->setStyleSheet("QLabel { background-color: red; color: white; }"); + } else { + peak_label->setStyleSheet(""); + } +} + +string get_bus_desc_label(const InputMapping::Bus &bus) +{ + string suffix; + if (bus.device.type == InputSourceType::ALSA_INPUT) { + ALSAPool::Device::State state = global_audio_mixer->get_alsa_card_state(bus.device.index); + if (state == ALSAPool::Device::State::STARTING) { + suffix = " (busy)"; + } else if (state == ALSAPool::Device::State::DEAD) { + suffix = " (dead)"; + } + } + + return bus.name + suffix; +} + +} // namespace + +MainWindow::MainWindow() + : ui(new Ui::MainWindow), midi_mapper(this) +{ + global_mainwindow = this; + ui->setupUi(this); + + global_disk_space_estimator = new DiskSpaceEstimator(bind(&MainWindow::report_disk_space, this, _1, _2)); + disk_free_label = new QLabel(this); + disk_free_label->setStyleSheet("QLabel {padding-right: 5px;}"); + ui->menuBar->setCornerWidget(disk_free_label); + + QActionGroup *audio_mapping_group = new QActionGroup(this); + ui->simple_audio_mode->setActionGroup(audio_mapping_group); + ui->multichannel_audio_mode->setActionGroup(audio_mapping_group); + + ui->me_live->set_output(Mixer::OUTPUT_LIVE); + ui->me_preview->set_output(Mixer::OUTPUT_PREVIEW); + + // The menus. + connect(ui->cut_action, &QAction::triggered, this, &MainWindow::cut_triggered); + connect(ui->exit_action, &QAction::triggered, this, &MainWindow::exit_triggered); + connect(ui->manual_action, &QAction::triggered, this, &MainWindow::manual_triggered); + connect(ui->about_action, &QAction::triggered, this, &MainWindow::about_triggered); + connect(ui->open_analyzer_action, &QAction::triggered, this, &MainWindow::open_analyzer_triggered); + connect(ui->simple_audio_mode, &QAction::triggered, this, &MainWindow::simple_audio_mode_triggered); + connect(ui->multichannel_audio_mode, &QAction::triggered, this, &MainWindow::multichannel_audio_mode_triggered); + connect(ui->input_mapping_action, &QAction::triggered, this, &MainWindow::input_mapping_triggered); + connect(ui->midi_mapping_action, &QAction::triggered, this, &MainWindow::midi_mapping_triggered); + connect(ui->timecode_stream_action, &QAction::triggered, this, &MainWindow::timecode_stream_triggered); + connect(ui->timecode_stdout_action, &QAction::triggered, this, &MainWindow::timecode_stdout_triggered); + + ui->timecode_stream_action->setChecked(global_flags.display_timecode_in_stream); + ui->timecode_stdout_action->setChecked(global_flags.display_timecode_on_stdout); + + if (global_flags.x264_video_to_http && isinf(global_flags.x264_crf)) { + connect(ui->x264_bitrate_action, &QAction::triggered, this, &MainWindow::x264_bitrate_triggered); + } else { + ui->x264_bitrate_action->setEnabled(false); + } + + connect(ui->video_menu, &QMenu::aboutToShow, [this]{ + fill_hdmi_sdi_output_device_menu(ui->hdmi_sdi_output_device_menu); + fill_hdmi_sdi_output_resolution_menu(ui->hdmi_sdi_output_resolution_menu); + }); + + // Hook up the transition buttons. (Keyboard shortcuts are set in set_transition_names().) + // TODO: Make them dynamic. + connect(ui->transition_btn1, &QPushButton::clicked, bind(&MainWindow::transition_clicked, this, 0)); + connect(ui->transition_btn2, &QPushButton::clicked, bind(&MainWindow::transition_clicked, this, 1)); + connect(ui->transition_btn3, &QPushButton::clicked, bind(&MainWindow::transition_clicked, this, 2)); + + // Aiee... + transition_btn1 = ui->transition_btn1; + transition_btn2 = ui->transition_btn2; + transition_btn3 = ui->transition_btn3; + qRegisterMetaType("std::string"); + qRegisterMetaType>("std::vector"); + connect(ui->me_live, &GLWidget::transition_names_updated, this, &MainWindow::set_transition_names); + qRegisterMetaType("Mixer::Output"); + + // Hook up the prev/next buttons on the audio views. + auto prev_page = [this]{ + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::MULTICHANNEL) { + ui->audio_views->setCurrentIndex((ui->audio_views->currentIndex() + 2) % 3); + } else { + ui->audio_views->setCurrentIndex(2 - ui->audio_views->currentIndex()); // Switch between 0 and 2. + } + }; + auto next_page = [this]{ + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::MULTICHANNEL) { + ui->audio_views->setCurrentIndex((ui->audio_views->currentIndex() + 1) % 3); + } else { + ui->audio_views->setCurrentIndex(2 - ui->audio_views->currentIndex()); // Switch between 0 and 2. + } + }; + connect(ui->compact_prev_page, &QAbstractButton::clicked, prev_page); + connect(ui->compact_next_page, &QAbstractButton::clicked, next_page); + connect(ui->full_prev_page, &QAbstractButton::clicked, prev_page); + connect(ui->full_next_page, &QAbstractButton::clicked, next_page); + connect(ui->video_grid_prev_page, &QAbstractButton::clicked, prev_page); + connect(ui->video_grid_next_page, &QAbstractButton::clicked, next_page); + + // And bind the same to PgUp/PgDown. + connect(new QShortcut(QKeySequence::MoveToNextPage, this), &QShortcut::activated, next_page); + connect(new QShortcut(QKeySequence::MoveToPreviousPage, this), &QShortcut::activated, prev_page); + + // When the audio view changes, move the previews. + connect(ui->audio_views, &QStackedWidget::currentChanged, bind(&MainWindow::audio_view_changed, this, _1)); + + if (global_flags.enable_quick_cut_keys) { + ui->quick_cut_enable_action->setChecked(true); + } + connect(ui->quick_cut_enable_action, &QAction::changed, [this](){ + global_flags.enable_quick_cut_keys = ui->quick_cut_enable_action->isChecked(); + }); + + last_audio_level_callback = steady_clock::now() - seconds(1); + + if (!global_flags.midi_mapping_filename.empty()) { + MIDIMappingProto midi_mapping; + if (!load_midi_mapping_from_file(global_flags.midi_mapping_filename, &midi_mapping)) { + fprintf(stderr, "Couldn't load MIDI mapping '%s'; exiting.\n", + global_flags.midi_mapping_filename.c_str()); + exit(1); + } + midi_mapper.set_midi_mapping(midi_mapping); + } + midi_mapper.refresh_highlights(); + midi_mapper.refresh_lights(); + if (global_flags.fullscreen) { + QMainWindow::showFullScreen(); + } +} + +void MainWindow::resizeEvent(QResizeEvent* event) +{ + QMainWindow::resizeEvent(event); + + // Ask for a relayout, but only after the event loop is done doing relayout + // on everything else. + QMetaObject::invokeMethod(this, "relayout", Qt::QueuedConnection); +} + +void MainWindow::mixer_created(Mixer *mixer) +{ + // Make the previews. + unsigned num_previews = mixer->get_num_channels(); + + const char qwerty[] = "QWERTYUIOP"; + for (unsigned i = 0; i < num_previews; ++i) { + Mixer::Output output = Mixer::Output(Mixer::OUTPUT_INPUT0 + i); + + QWidget *preview = new QWidget(this); // Will be connected to a layout immediately after the loop. + Ui::Display *ui_display = new Ui::Display; + ui_display->setupUi(preview); + ui_display->label->setText(mixer->get_channel_name(output).c_str()); + ui_display->display->set_output(output); + previews.push_back(ui_display); + + // Hook up the click. + connect(ui_display->display, &GLWidget::clicked, bind(&MainWindow::channel_clicked, this, i)); + + // Let the theme update the text whenever the resolution or color changed. + connect(ui_display->display, &GLWidget::name_updated, this, &MainWindow::update_channel_name); + connect(ui_display->display, &GLWidget::color_updated, this, &MainWindow::update_channel_color); + + // Hook up the keyboard key. + QShortcut *shortcut = nullptr; + if (i < 9) { + shortcut = new QShortcut(QKeySequence(Qt::Key_1 + i), this); + } else if (i == 9) { + shortcut = new QShortcut(QKeySequence(Qt::Key_0), this); + } + if (shortcut != nullptr) { + connect(shortcut, &QShortcut::activated, bind(&MainWindow::channel_clicked, this, i)); + } + + // Hook up the quick-cut key. + if (i < strlen(qwerty)) { + QShortcut *shortcut = new QShortcut(QKeySequence(qwerty[i]), this); + connect(shortcut, &QShortcut::activated, bind(&MainWindow::quick_cut_activated, this, i)); + } + + // Hook up the white balance button (irrelevant if invisible). + ui_display->wb_button->setVisible(mixer->get_supports_set_wb(output)); + connect(ui_display->wb_button, &QPushButton::clicked, bind(&MainWindow::wb_button_clicked, this, i)); + } + + // Connect the previews to the correct layout. + audio_view_changed(ui->audio_views->currentIndex()); + + global_audio_mixer->set_state_changed_callback(bind(&MainWindow::audio_state_changed, this)); + + slave_knob(ui->locut_cutoff_knob, ui->locut_cutoff_knob_2); + slave_knob(ui->limiter_threshold_knob, ui->limiter_threshold_knob_2); + slave_knob(ui->makeup_gain_knob, ui->makeup_gain_knob_2); + slave_checkbox(ui->makeup_gain_auto_checkbox, ui->makeup_gain_auto_checkbox_2); + slave_checkbox(ui->limiter_enabled, ui->limiter_enabled_2); + + reset_audio_mapping_ui(); + + // TODO: Fetch all of the values these for completeness, + // not just the enable knobs implied by flags. + ui->limiter_enabled->setChecked(global_audio_mixer->get_limiter_enabled()); + ui->makeup_gain_auto_checkbox->setChecked(global_audio_mixer->get_final_makeup_gain_auto()); + + // Controls used only for simple audio fetch their state from the first bus. + constexpr unsigned simple_bus_index = 0; + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::SIMPLE) { + ui->locut_enabled->setChecked(global_audio_mixer->get_locut_enabled(simple_bus_index)); + ui->gainstaging_knob->setValue(global_audio_mixer->get_gain_staging_db(simple_bus_index)); + ui->gainstaging_auto_checkbox->setChecked(global_audio_mixer->get_gain_staging_auto(simple_bus_index)); + ui->compressor_enabled->setChecked(global_audio_mixer->get_compressor_enabled(simple_bus_index)); + ui->compressor_threshold_db_display->setText( + QString::fromStdString(format_db(mixer->get_audio_mixer()->get_compressor_threshold_dbfs(simple_bus_index), DB_WITH_SIGN))); + } + connect(ui->locut_enabled, &QCheckBox::stateChanged, [this](int state){ + global_audio_mixer->set_locut_enabled(simple_bus_index, state == Qt::Checked); + midi_mapper.refresh_lights(); + }); + connect(ui->gainstaging_knob, &QAbstractSlider::valueChanged, + bind(&MainWindow::gain_staging_knob_changed, this, simple_bus_index, _1)); + connect(ui->gainstaging_auto_checkbox, &QCheckBox::stateChanged, [this](int state){ + global_audio_mixer->set_gain_staging_auto(simple_bus_index, state == Qt::Checked); + midi_mapper.refresh_lights(); + }); + connect(ui->compressor_threshold_knob, &QDial::valueChanged, + bind(&MainWindow::compressor_threshold_knob_changed, this, simple_bus_index, _1)); + connect(ui->compressor_enabled, &QCheckBox::stateChanged, [this](int state){ + global_audio_mixer->set_compressor_enabled(simple_bus_index, state == Qt::Checked); + midi_mapper.refresh_lights(); + }); + + // Global mastering controls. + QString limiter_threshold_label( + QString::fromStdString(format_db(mixer->get_audio_mixer()->get_limiter_threshold_dbfs(), DB_WITH_SIGN))); + ui->limiter_threshold_db_display->setText(limiter_threshold_label); + ui->limiter_threshold_db_display_2->setText(limiter_threshold_label); + + connect(ui->locut_cutoff_knob, &QDial::valueChanged, this, &MainWindow::cutoff_knob_changed); + cutoff_knob_changed(ui->locut_cutoff_knob->value()); + + connect(ui->makeup_gain_knob, &QAbstractSlider::valueChanged, this, &MainWindow::final_makeup_gain_knob_changed); + connect(ui->makeup_gain_auto_checkbox, &QCheckBox::stateChanged, [this](int state){ + global_audio_mixer->set_final_makeup_gain_auto(state == Qt::Checked); + midi_mapper.refresh_lights(); + }); + + connect(ui->limiter_threshold_knob, &QDial::valueChanged, this, &MainWindow::limiter_threshold_knob_changed); + connect(ui->limiter_enabled, &QCheckBox::stateChanged, [this](int state){ + global_audio_mixer->set_limiter_enabled(state == Qt::Checked); + midi_mapper.refresh_lights(); + }); + connect(ui->reset_meters_button, &QPushButton::clicked, this, &MainWindow::reset_meters_button_clicked); + // Even though we have a reset button right next to it, the fact that + // the expanded audio view labels are clickable makes it natural to + // click this one as well. + connect(ui->peak_display, &ClickableLabel::clicked, this, &MainWindow::reset_meters_button_clicked); + mixer->get_audio_mixer()->set_audio_level_callback(bind(&MainWindow::audio_level_callback, this, _1, _2, _3, _4, _5, _6, _7, _8)); + + midi_mapper.refresh_highlights(); + midi_mapper.refresh_lights(); + midi_mapper.start_thread(); + + analyzer.reset(new Analyzer); + + global_mixer->set_theme_menu_callback(bind(&MainWindow::setup_theme_menu, this)); + setup_theme_menu(); + + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = schedule_cut_signal; + act.sa_flags = SA_RESTART; + sigaction(SIGHUP, &act, nullptr); + + // Mostly for debugging. Don't override SIGINT, that's so evil if + // shutdown isn't instant. + memset(&act, 0, sizeof(act)); + act.sa_handler = quit_signal; + act.sa_flags = SA_RESTART; + sigaction(SIGUSR1, &act, nullptr); +} + +void MainWindow::reset_audio_mapping_ui() +{ + bool simple = (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::SIMPLE); + + ui->simple_audio_mode->setChecked(simple); + ui->multichannel_audio_mode->setChecked(!simple); + ui->input_mapping_action->setEnabled(!simple); + ui->midi_mapping_action->setEnabled(!simple); + + ui->locut_enabled->setVisible(simple); + ui->gainstaging_label->setVisible(simple); + ui->gainstaging_knob->setVisible(simple); + ui->gainstaging_db_display->setVisible(simple); + ui->gainstaging_auto_checkbox->setVisible(simple); + ui->compressor_threshold_label->setVisible(simple); + ui->compressor_threshold_knob->setVisible(simple); + ui->compressor_threshold_db_display->setVisible(simple); + ui->compressor_enabled->setVisible(simple); + + setup_audio_miniview(); + setup_audio_expanded_view(); + + if (simple) { + ui->compact_label->setText("Compact audio view (1/2) "); + ui->video_grid_label->setText("Video grid display (2/2) "); + if (ui->audio_views->currentIndex() == 1) { + // Full audio view is not available in simple mode. + ui->audio_views->setCurrentIndex(0); + } + } else { + ui->compact_label->setText("Compact audio view (1/3) "); + ui->full_label->setText("Full audio view (2/3) "); + ui->video_grid_label->setText("Video grid display (3/3) "); + } + + midi_mapper.refresh_highlights(); + midi_mapper.refresh_lights(); +} + +void MainWindow::setup_audio_miniview() +{ + // Remove any existing channels. + for (QLayoutItem *item; (item = ui->faders->takeAt(0)) != nullptr; ) { + delete item->widget(); + delete item; + } + audio_miniviews.clear(); + + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::SIMPLE) { + return; + } + + // Set up brand new ones from the input mapping. + InputMapping mapping = global_audio_mixer->get_input_mapping(); + audio_miniviews.resize(mapping.buses.size()); + for (unsigned bus_index = 0; bus_index < mapping.buses.size(); ++bus_index) { + QWidget *channel = new QWidget(this); + Ui::AudioMiniView *ui_audio_miniview = new Ui::AudioMiniView; + ui_audio_miniview->setupUi(channel); + ui_audio_miniview->bus_desc_label->setFullText( + QString::fromStdString(get_bus_desc_label(mapping.buses[bus_index]))); + audio_miniviews[bus_index] = ui_audio_miniview; + + // Set up the peak meter. + VUMeter *peak_meter = ui_audio_miniview->peak_meter; + peak_meter->set_min_level(-30.0f); + peak_meter->set_max_level(0.0f); + peak_meter->set_ref_level(0.0f); + + ui_audio_miniview->fader->setDbValue(global_audio_mixer->get_fader_volume(bus_index)); + + ui->faders->addWidget(channel); + + connect(ui_audio_miniview->fader, &NonLinearFader::dbValueChanged, + bind(&MainWindow::mini_fader_changed, this, bus_index, _1)); + connect(ui_audio_miniview->peak_display_label, &ClickableLabel::clicked, + [bus_index]() { + global_audio_mixer->reset_peak(bus_index); + }); + } +} + +void MainWindow::setup_audio_expanded_view() +{ + // Remove any existing channels. + for (QLayoutItem *item; (item = ui->buses->takeAt(0)) != nullptr; ) { + delete item->widget(); + delete item; + } + audio_expanded_views.clear(); + + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::SIMPLE) { + return; + } + + // Set up brand new ones from the input mapping. + InputMapping mapping = global_audio_mixer->get_input_mapping(); + audio_expanded_views.resize(mapping.buses.size()); + for (unsigned bus_index = 0; bus_index < mapping.buses.size(); ++bus_index) { + QWidget *channel = new QWidget(this); + Ui::AudioExpandedView *ui_audio_expanded_view = new Ui::AudioExpandedView; + ui_audio_expanded_view->setupUi(channel); + ui_audio_expanded_view->bus_desc_label->setFullText( + QString::fromStdString(get_bus_desc_label(mapping.buses[bus_index]))); + audio_expanded_views[bus_index] = ui_audio_expanded_view; + update_stereo_knob_and_label(bus_index, lrintf(100.0f * global_audio_mixer->get_stereo_width(bus_index))); + update_eq_label(bus_index, EQ_BAND_TREBLE, global_audio_mixer->get_eq(bus_index, EQ_BAND_TREBLE)); + update_eq_label(bus_index, EQ_BAND_MID, global_audio_mixer->get_eq(bus_index, EQ_BAND_MID)); + update_eq_label(bus_index, EQ_BAND_BASS, global_audio_mixer->get_eq(bus_index, EQ_BAND_BASS)); + ui_audio_expanded_view->fader->setDbValue(global_audio_mixer->get_fader_volume(bus_index)); + ui_audio_expanded_view->mute_button->setChecked(global_audio_mixer->get_mute(bus_index)); + connect(ui_audio_expanded_view->mute_button, &QPushButton::toggled, + bind(&MainWindow::mute_button_toggled, this, bus_index, _1)); + ui->buses->addWidget(channel); + + ui_audio_expanded_view->locut_enabled->setChecked(global_audio_mixer->get_locut_enabled(bus_index)); + connect(ui_audio_expanded_view->locut_enabled, &QCheckBox::stateChanged, [this, bus_index](int state){ + global_audio_mixer->set_locut_enabled(bus_index, state == Qt::Checked); + midi_mapper.refresh_lights(); + }); + + connect(ui_audio_expanded_view->stereo_width_knob, &QDial::valueChanged, + bind(&MainWindow::stereo_width_knob_changed, this, bus_index, _1)); + + connect(ui_audio_expanded_view->treble_knob, &QDial::valueChanged, + bind(&MainWindow::eq_knob_changed, this, bus_index, EQ_BAND_TREBLE, _1)); + connect(ui_audio_expanded_view->mid_knob, &QDial::valueChanged, + bind(&MainWindow::eq_knob_changed, this, bus_index, EQ_BAND_MID, _1)); + connect(ui_audio_expanded_view->bass_knob, &QDial::valueChanged, + bind(&MainWindow::eq_knob_changed, this, bus_index, EQ_BAND_BASS, _1)); + + ui_audio_expanded_view->gainstaging_knob->setValue(global_audio_mixer->get_gain_staging_db(bus_index)); + ui_audio_expanded_view->gainstaging_auto_checkbox->setChecked(global_audio_mixer->get_gain_staging_auto(bus_index)); + ui_audio_expanded_view->compressor_enabled->setChecked(global_audio_mixer->get_compressor_enabled(bus_index)); + + connect(ui_audio_expanded_view->gainstaging_knob, &QAbstractSlider::valueChanged, bind(&MainWindow::gain_staging_knob_changed, this, bus_index, _1)); + connect(ui_audio_expanded_view->gainstaging_auto_checkbox, &QCheckBox::stateChanged, [this, bus_index](int state){ + global_audio_mixer->set_gain_staging_auto(bus_index, state == Qt::Checked); + midi_mapper.refresh_lights(); + }); + + connect(ui_audio_expanded_view->compressor_threshold_knob, &QDial::valueChanged, bind(&MainWindow::compressor_threshold_knob_changed, this, bus_index, _1)); + connect(ui_audio_expanded_view->compressor_enabled, &QCheckBox::stateChanged, [this, bus_index](int state){ + global_audio_mixer->set_compressor_enabled(bus_index, state == Qt::Checked); + midi_mapper.refresh_lights(); + }); + + slave_fader(audio_miniviews[bus_index]->fader, ui_audio_expanded_view->fader); + + // Set up the peak meter. + VUMeter *peak_meter = ui_audio_expanded_view->peak_meter; + peak_meter->set_min_level(-30.0f); + peak_meter->set_max_level(0.0f); + peak_meter->set_ref_level(0.0f); + + connect(ui_audio_expanded_view->peak_display_label, &ClickableLabel::clicked, + [this, bus_index]() { + global_audio_mixer->reset_peak(bus_index); + midi_mapper.refresh_lights(); + }); + } + + update_cutoff_labels(global_audio_mixer->get_locut_cutoff()); +} + +void MainWindow::mixer_shutting_down() +{ + ui->me_live->shutdown(); + ui->me_preview->shutdown(); + + for (Ui::Display *display : previews) { + display->display->shutdown(); + } + + analyzer->mixer_shutting_down(); +} + +void MainWindow::cut_triggered() +{ + global_mixer->schedule_cut(); +} + +void MainWindow::x264_bitrate_triggered() +{ + bool ok; + int new_bitrate = QInputDialog::getInt(this, "Change x264 bitrate", "Choose new bitrate for x264 HTTP output (from 100–100,000 kbit/sec):", global_flags.x264_bitrate, /*min=*/100, /*max=*/100000, /*step=*/100, &ok); + if (ok && new_bitrate >= 100 && new_bitrate <= 100000) { + global_flags.x264_bitrate = new_bitrate; + global_mixer->change_x264_bitrate(new_bitrate); + } +} + +void MainWindow::exit_triggered() +{ + close(); +} + +void MainWindow::manual_triggered() +{ + if (!QDesktopServices::openUrl(QUrl("https://nageru.sesse.net/doc/"))) { + QMessageBox msgbox; + msgbox.setText("Could not launch manual in web browser.\nPlease see https://nageru.sesse.net/doc/ manually."); + msgbox.exec(); + } +} + +void MainWindow::about_triggered() +{ + AboutDialog().exec(); +} + +void MainWindow::open_analyzer_triggered() +{ + analyzer->show(); +} + +void MainWindow::simple_audio_mode_triggered() +{ + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::SIMPLE) { + return; + } + unsigned card_index = global_audio_mixer->get_simple_input(); + if (card_index == numeric_limits::max()) { + QMessageBox::StandardButton reply = + QMessageBox::question(this, + "Mapping too complex", + "The current audio mapping is too complicated to be representable in simple mode, " + "and will be discarded if you proceed. Really go to simple audio mode?", + QMessageBox::Yes | QMessageBox::No); + if (reply == QMessageBox::No) { + ui->simple_audio_mode->setChecked(false); + ui->multichannel_audio_mode->setChecked(true); + return; + } + card_index = 0; + } + global_audio_mixer->set_simple_input(/*card_index=*/card_index); + reset_audio_mapping_ui(); +} + +void MainWindow::multichannel_audio_mode_triggered() +{ + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::MULTICHANNEL) { + return; + } + + // Take the generated input mapping from the simple input, + // and set it as a normal multichannel mapping, which causes + // the mode to go to multichannel. + global_audio_mixer->set_input_mapping(global_audio_mixer->get_input_mapping()); + reset_audio_mapping_ui(); +} + +void MainWindow::input_mapping_triggered() +{ + if (InputMappingDialog().exec() == QDialog::Accepted) { + setup_audio_miniview(); + setup_audio_expanded_view(); + } + midi_mapper.refresh_highlights(); + midi_mapper.refresh_lights(); +} + +void MainWindow::midi_mapping_triggered() +{ + MIDIMappingDialog(&midi_mapper).exec(); +} + +void MainWindow::timecode_stream_triggered() +{ + global_mixer->set_display_timecode_in_stream(ui->timecode_stream_action->isChecked()); +} + +void MainWindow::timecode_stdout_triggered() +{ + global_mixer->set_display_timecode_on_stdout(ui->timecode_stdout_action->isChecked()); +} + +void MainWindow::gain_staging_knob_changed(unsigned bus_index, int value) +{ + if (bus_index == 0) { + ui->gainstaging_auto_checkbox->setCheckState(Qt::Unchecked); + } + if (bus_index < audio_expanded_views.size()) { + audio_expanded_views[bus_index]->gainstaging_auto_checkbox->setCheckState(Qt::Unchecked); + } + + float gain_db = value * 0.1f; + global_audio_mixer->set_gain_staging_db(bus_index, gain_db); + + // The label will be updated by the audio level callback. +} + +void MainWindow::final_makeup_gain_knob_changed(int value) +{ + ui->makeup_gain_auto_checkbox->setCheckState(Qt::Unchecked); + + float gain_db = value * 0.1f; + global_audio_mixer->set_final_makeup_gain_db(gain_db); + + // The label will be updated by the audio level callback. +} + +void MainWindow::cutoff_knob_changed(int value) +{ + float octaves = value * 0.1f; + float cutoff_hz = 20.0 * pow(2.0, octaves); + global_audio_mixer->set_locut_cutoff(cutoff_hz); + update_cutoff_labels(cutoff_hz); +} + +void MainWindow::update_cutoff_labels(float cutoff_hz) +{ + char buf[256]; + snprintf(buf, sizeof(buf), "%ld Hz", lrintf(cutoff_hz)); + ui->locut_cutoff_display->setText(buf); + ui->locut_cutoff_display_2->setText(buf); + + for (unsigned bus_index = 0; bus_index < audio_expanded_views.size(); ++bus_index) { + audio_expanded_views[bus_index]->locut_enabled->setText( + QString("Lo-cut: ") + buf); + } +} + +void MainWindow::report_disk_space(off_t free_bytes, double estimated_seconds_left) +{ + char time_str[256]; + if (estimated_seconds_left < 60.0) { + strcpy(time_str, "Less than a minute"); + } else if (estimated_seconds_left < 1800.0) { // Less than half an hour: Xm Ys (red). + int s = lrintf(estimated_seconds_left); + int m = s / 60; + s %= 60; + snprintf(time_str, sizeof(time_str), "%dm %ds", m, s); + } else if (estimated_seconds_left < 3600.0) { // Less than an hour: Xm. + int m = lrintf(estimated_seconds_left / 60.0); + snprintf(time_str, sizeof(time_str), "%dm", m); + } else if (estimated_seconds_left < 36000.0) { // Less than ten hours: Xh Ym. + int m = lrintf(estimated_seconds_left / 60.0); + int h = m / 60; + m %= 60; + snprintf(time_str, sizeof(time_str), "%dh %dm", h, m); + } else { // More than ten hours: Xh. + int h = lrintf(estimated_seconds_left / 3600.0); + snprintf(time_str, sizeof(time_str), "%dh", h); + } + char buf[256]; + snprintf(buf, sizeof(buf), "Disk free: %'.0f MB (approx. %s)", free_bytes / 1048576.0, time_str); + + std::string label = buf; + + post_to_main_thread([this, label]{ + disk_free_label->setText(QString::fromStdString(label)); + ui->menuBar->setCornerWidget(disk_free_label); // Need to set this again for the sizing to get right. + }); +} + +void MainWindow::stereo_width_knob_changed(unsigned bus_index, int value) +{ + float stereo_width = value * 0.01f; + global_audio_mixer->set_stereo_width(bus_index, stereo_width); + + update_stereo_label(bus_index, value); +} + +void MainWindow::eq_knob_changed(unsigned bus_index, EQBand band, int value) +{ + float gain_db = value * 0.1f; + global_audio_mixer->set_eq(bus_index, band, gain_db); + + update_eq_label(bus_index, band, gain_db); +} + +void MainWindow::update_stereo_knob_and_label(unsigned bus_index, int stereo_width_percent) +{ + Ui::AudioExpandedView *view = audio_expanded_views[bus_index]; + + if (global_audio_mixer->is_mono(bus_index)) { + view->stereo_width_knob->setEnabled(false); + view->stereo_width_label->setEnabled(false); + } else { + view->stereo_width_knob->setEnabled(true); + view->stereo_width_label->setEnabled(true); + } + view->stereo_width_knob->setValue(stereo_width_percent); + update_stereo_label(bus_index, stereo_width_percent); +} + +void MainWindow::update_stereo_label(unsigned bus_index, int stereo_width_percent) +{ + Ui::AudioExpandedView *view = audio_expanded_views[bus_index]; + + if (global_audio_mixer->is_mono(bus_index)) { + view->stereo_width_label->setText("Mono"); + } else { + char buf[256]; + snprintf(buf, sizeof(buf), "Stereo: %d%%", stereo_width_percent); + view->stereo_width_label->setText(buf); + } +} + +void MainWindow::update_eq_label(unsigned bus_index, EQBand band, float gain_db) +{ + Ui::AudioExpandedView *view = audio_expanded_views[bus_index]; + string db_string = format_db(gain_db, DB_WITH_SIGN); + switch (band) { + case EQ_BAND_TREBLE: + view->treble_label->setText(QString::fromStdString("Treble: " + db_string)); + break; + case EQ_BAND_MID: + view->mid_label->setText(QString::fromStdString("Mid: " + db_string)); + break; + case EQ_BAND_BASS: + view->bass_label->setText(QString::fromStdString("Bass: " + db_string)); + break; + default: + assert(false); + } +} + +void MainWindow::setup_theme_menu() +{ + std::vector theme_menu_entries = global_mixer->get_theme_menu(); + + if (theme_menu != nullptr) { + ui->menuBar->removeAction(theme_menu->menuAction()); + theme_menu = nullptr; + } + + if (!theme_menu_entries.empty()) { + theme_menu = new QMenu("&Theme"); + for (const Theme::MenuEntry &entry : theme_menu_entries) { + QAction *action = theme_menu->addAction(QString::fromStdString(entry.text)); + connect(action, &QAction::triggered, [entry] { + global_mixer->theme_menu_entry_clicked(entry.lua_ref); + }); + } + ui->menuBar->insertMenu(ui->menu_Help->menuAction(), theme_menu); + } +} + +void MainWindow::limiter_threshold_knob_changed(int value) +{ + float threshold_dbfs = value * 0.1f; + global_audio_mixer->set_limiter_threshold_dbfs(threshold_dbfs); + ui->limiter_threshold_db_display->setText( + QString::fromStdString(format_db(threshold_dbfs, DB_WITH_SIGN))); + ui->limiter_threshold_db_display_2->setText( + QString::fromStdString(format_db(threshold_dbfs, DB_WITH_SIGN))); +} + +void MainWindow::compressor_threshold_knob_changed(unsigned bus_index, int value) +{ + float threshold_dbfs = value * 0.1f; + global_audio_mixer->set_compressor_threshold_dbfs(bus_index, threshold_dbfs); + + QString label(QString::fromStdString(format_db(threshold_dbfs, DB_WITH_SIGN))); + if (bus_index == 0) { + ui->compressor_threshold_db_display->setText(label); + } + if (bus_index < audio_expanded_views.size()) { + audio_expanded_views[bus_index]->compressor_threshold_db_display->setText(label); + } +} + +void MainWindow::mini_fader_changed(int bus, double volume_db) +{ + QString label(QString::fromStdString(format_db(volume_db, DB_WITH_SIGN))); + audio_miniviews[bus]->fader_label->setText(label); + audio_expanded_views[bus]->fader_label->setText(label); + + global_audio_mixer->set_fader_volume(bus, volume_db); +} + +void MainWindow::mute_button_toggled(int bus, bool checked) +{ + global_audio_mixer->set_mute(bus, checked); + midi_mapper.refresh_lights(); +} + +void MainWindow::reset_meters_button_clicked() +{ + global_audio_mixer->reset_meters(); + ui->peak_display->setText(QString::fromStdString(format_db(-HUGE_VAL, DB_WITH_SIGN | DB_BARE))); + ui->peak_display->setStyleSheet(""); +} + +void MainWindow::audio_level_callback(float level_lufs, float peak_db, vector bus_levels, + float global_level_lufs, + float range_low_lufs, float range_high_lufs, + float final_makeup_gain_db, + float correlation) +{ + steady_clock::time_point now = steady_clock::now(); + + // The meters are somewhat inefficient to update. Only update them + // every 100 ms or so (we get updates every 5–20 ms). Note that this + // means that the digital peak meters are ever so slightly too low + // (each update won't be a faithful representation of the highest peak + // since the previous update, since there are frames we won't draw), + // but the _peak_ of the peak meters will be correct (it's tracked in + // AudioMixer, not here), and that's much more important. + double last_update_age = duration(now - last_audio_level_callback).count(); + if (last_update_age < 0.100) { + return; + } + last_audio_level_callback = now; + + post_to_main_thread([=]() { + ui->vu_meter->set_level(level_lufs); + for (unsigned bus_index = 0; bus_index < bus_levels.size(); ++bus_index) { + if (bus_index < audio_miniviews.size()) { + const AudioMixer::BusLevel &level = bus_levels[bus_index]; + Ui::AudioMiniView *miniview = audio_miniviews[bus_index]; + miniview->peak_meter->set_level( + level.current_level_dbfs[0], level.current_level_dbfs[1]); + miniview->peak_meter->set_peak( + level.peak_level_dbfs[0], level.peak_level_dbfs[1]); + set_peak_label(miniview->peak_display_label, level.historic_peak_dbfs); + + Ui::AudioExpandedView *view = audio_expanded_views[bus_index]; + view->peak_meter->set_level( + level.current_level_dbfs[0], level.current_level_dbfs[1]); + view->peak_meter->set_peak( + level.peak_level_dbfs[0], level.peak_level_dbfs[1]); + view->reduction_meter->set_reduction_db(level.compressor_attenuation_db); + view->gainstaging_knob->blockSignals(true); + view->gainstaging_knob->setValue(lrintf(level.gain_staging_db * 10.0f)); + view->gainstaging_knob->blockSignals(false); + view->gainstaging_db_display->setText( + QString("Gain: ") + + QString::fromStdString(format_db(level.gain_staging_db, DB_WITH_SIGN))); + set_peak_label(view->peak_display_label, level.historic_peak_dbfs); + + midi_mapper.set_has_peaked(bus_index, level.historic_peak_dbfs >= -0.1f); + } + } + ui->lra_meter->set_levels(global_level_lufs, range_low_lufs, range_high_lufs); + ui->correlation_meter->set_correlation(correlation); + + ui->peak_display->setText(QString::fromStdString(format_db(peak_db, DB_BARE))); + set_peak_label(ui->peak_display, peak_db); + + // NOTE: Will be invisible when using multitrack audio. + if (!bus_levels.empty()) { + ui->gainstaging_knob->blockSignals(true); + ui->gainstaging_knob->setValue(lrintf(bus_levels[0].gain_staging_db * 10.0f)); + ui->gainstaging_knob->blockSignals(false); + ui->gainstaging_db_display->setText( + QString::fromStdString(format_db(bus_levels[0].gain_staging_db, DB_WITH_SIGN))); + } + + ui->makeup_gain_knob->blockSignals(true); + ui->makeup_gain_knob->setValue(lrintf(final_makeup_gain_db * 10.0f)); + ui->makeup_gain_knob->blockSignals(false); + ui->makeup_gain_db_display->setText( + QString::fromStdString(format_db(final_makeup_gain_db, DB_WITH_SIGN))); + ui->makeup_gain_db_display_2->setText( + QString::fromStdString(format_db(final_makeup_gain_db, DB_WITH_SIGN))); + + // Peak labels could have changed. + midi_mapper.refresh_lights(); + }); +} + +void MainWindow::relayout() +{ + int height = ui->vertical_layout->geometry().height(); + if (height <= 0) { + // Seemingly this can happen and must be ignored. + return; + } + + double remaining_height = height; + + // Allocate the height; the most important part is to keep the main displays + // at the right aspect if at all possible. + double me_width = ui->me_preview->width(); + double me_height = me_width * double(global_flags.height) / double(global_flags.width) + ui->label_preview->height() + ui->preview_vertical_layout->spacing(); + + // TODO: Scale the widths when we need to do this. + if (me_height / double(height) > 0.8) { + me_height = height * 0.8; + } + remaining_height -= me_height + ui->vertical_layout->spacing(); + + // Space between the M/E displays and the audio strip. + remaining_height -= ui->vertical_layout->spacing(); + + // The label above the audio strip. + double compact_label_height = ui->compact_label->minimumHeight() + + ui->compact_audio_layout->spacing(); + remaining_height -= compact_label_height; + + // The previews will be constrained by the remaining height, and the width. + double preview_label_height = previews[0]->label->minimumSize().height() + + previews[0]->main_vertical_layout->spacing(); + int preview_total_width = ui->preview_displays->geometry().width() - (previews.size() - 1) * ui->preview_displays->spacing(); + double preview_height = min(remaining_height - preview_label_height, (preview_total_width / double(previews.size())) * double(global_flags.height) / double(global_flags.width)); + remaining_height -= preview_height + preview_label_height + ui->vertical_layout->spacing(); + + ui->vertical_layout->setStretch(0, lrintf(me_height)); + ui->vertical_layout->setStretch(1, + lrintf(compact_label_height) + + lrintf(remaining_height) + + lrintf(preview_height + preview_label_height)); // Audio strip and previews together. + + ui->compact_audio_layout->setStretch(0, lrintf(compact_label_height)); + ui->compact_audio_layout->setStretch(1, lrintf(remaining_height)); // Audio strip. + ui->compact_audio_layout->setStretch(2, lrintf(preview_height + preview_label_height)); + + if (current_audio_view == 0) { // Compact audio view. + // Set the widths for the previews. + double preview_width = preview_height * double(global_flags.width) / double(global_flags.height); + for (unsigned i = 0; i < previews.size(); ++i) { + ui->preview_displays->setStretch(i, lrintf(preview_width)); + } + + // The preview horizontal spacer. + double remaining_preview_width = preview_total_width - previews.size() * preview_width; + ui->preview_displays->setStretch(previews.size(), lrintf(remaining_preview_width)); + } else if (current_audio_view == 2) { // Video grid view. + // QGridLayout doesn't do it for us, since we need to be able to remove rows + // or columns as the grid changes, and it won't do that. Thus, position everything + // by hand. + constexpr int spacing = 6; + int grid_width = ui->preview_displays_grid->geometry().width(); + int grid_height = ui->preview_displays_grid->geometry().height(); + int best_preview_width = 0; + unsigned best_num_rows = 1, best_num_cols = 1; + for (unsigned num_rows = 1; num_rows <= previews.size(); ++num_rows) { + int num_cols = (previews.size() + num_rows - 1) / num_rows; + + int max_preview_height = (grid_height - spacing * (num_rows - 1)) / num_rows - preview_label_height; + int max_preview_width = (grid_width - spacing * (num_cols - 1)) / num_cols; + int preview_width = std::min(max_preview_width, max_preview_height * double(global_flags.width) / double(global_flags.height)); + + if (preview_width > best_preview_width) { + best_preview_width = preview_width; + best_num_rows = num_rows; + best_num_cols = num_cols; + } + } + + double cell_height = lrintf(best_preview_width * double(global_flags.height) / double(global_flags.width)) + preview_label_height; + remaining_height = grid_height - best_num_rows * cell_height - (best_num_rows - 1) * spacing; + int cell_width = best_preview_width; + int remaining_width = grid_width - best_num_cols * cell_width - (best_num_cols - 1) * spacing; + + for (unsigned i = 0; i < previews.size(); ++i) { + int col_idx = i % best_num_cols; + int row_idx = i / best_num_cols; + + double top = remaining_height * 0.5f + row_idx * (cell_height + spacing); + double bottom = top + cell_height; + double left = remaining_width * 0.5f + col_idx * (cell_width + spacing); + double right = left + cell_width; + + QRect rect; + rect.setTop(lrintf(top)); + rect.setBottom(lrintf(bottom)); + rect.setLeft(lrintf(left)); + rect.setRight(lrintf(right)); + + QWidget *display = static_cast(previews[i]->frame->parent()); + display->setGeometry(rect); + display->show(); + } + } +} + +void MainWindow::set_locut(float value) +{ + set_relative_value(ui->locut_cutoff_knob, value); +} + +void MainWindow::set_limiter_threshold(float value) +{ + set_relative_value(ui->limiter_threshold_knob, value); +} + +void MainWindow::set_makeup_gain(float value) +{ + set_relative_value(ui->makeup_gain_knob, value); +} + +void MainWindow::set_stereo_width(unsigned bus_idx, float value) +{ + set_relative_value_if_exists(bus_idx, &Ui::AudioExpandedView::stereo_width_knob, value); +} + +void MainWindow::set_treble(unsigned bus_idx, float value) +{ + set_relative_value_if_exists(bus_idx, &Ui::AudioExpandedView::treble_knob, value); +} + +void MainWindow::set_mid(unsigned bus_idx, float value) +{ + set_relative_value_if_exists(bus_idx, &Ui::AudioExpandedView::mid_knob, value); +} + +void MainWindow::set_bass(unsigned bus_idx, float value) +{ + set_relative_value_if_exists(bus_idx, &Ui::AudioExpandedView::bass_knob, value); +} + +void MainWindow::set_gain(unsigned bus_idx, float value) +{ + set_relative_value_if_exists(bus_idx, &Ui::AudioExpandedView::gainstaging_knob, value); +} + +void MainWindow::set_compressor_threshold(unsigned bus_idx, float value) +{ + set_relative_value_if_exists(bus_idx, &Ui::AudioExpandedView::compressor_threshold_knob, value); +} + +void MainWindow::set_fader(unsigned bus_idx, float value) +{ + set_relative_value_if_exists(bus_idx, &Ui::AudioExpandedView::fader, value); +} + +void MainWindow::toggle_mute(unsigned bus_idx) +{ + click_button_if_exists(bus_idx, &Ui::AudioExpandedView::mute_button); +} + +void MainWindow::toggle_locut(unsigned bus_idx) +{ + click_button_if_exists(bus_idx, &Ui::AudioExpandedView::locut_enabled); +} + +void MainWindow::toggle_auto_gain_staging(unsigned bus_idx) +{ + click_button_if_exists(bus_idx, &Ui::AudioExpandedView::gainstaging_auto_checkbox); +} + +void MainWindow::toggle_compressor(unsigned bus_idx) +{ + click_button_if_exists(bus_idx, &Ui::AudioExpandedView::compressor_enabled); +} + +void MainWindow::clear_peak(unsigned bus_idx) +{ + post_to_main_thread([=]{ + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::MULTICHANNEL) { + global_audio_mixer->reset_peak(bus_idx); + midi_mapper.set_has_peaked(bus_idx, false); + midi_mapper.refresh_lights(); + } + }); +} + +void MainWindow::clear_all_highlights() +{ + post_to_main_thread([this]{ + highlight_locut(false); + highlight_limiter_threshold(false); + highlight_makeup_gain(false); + highlight_toggle_limiter(false); + highlight_toggle_auto_makeup_gain(false); + for (unsigned bus_idx = 0; bus_idx < audio_expanded_views.size(); ++bus_idx) { + highlight_treble(bus_idx, false); + highlight_mid(bus_idx, false); + highlight_bass(bus_idx, false); + highlight_gain(bus_idx, false); + highlight_compressor_threshold(bus_idx, false); + highlight_fader(bus_idx, false); + highlight_mute(bus_idx, false); + highlight_toggle_locut(bus_idx, false); + highlight_toggle_auto_gain_staging(bus_idx, false); + highlight_toggle_compressor(bus_idx, false); + } + }); +} + +void MainWindow::toggle_limiter() +{ + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::MULTICHANNEL) { + ui->limiter_enabled->click(); + } +} + +void MainWindow::toggle_auto_makeup_gain() +{ + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::MULTICHANNEL) { + ui->makeup_gain_auto_checkbox->click(); + } +} + +void MainWindow::highlight_locut(bool highlight) +{ + post_to_main_thread([this, highlight]{ + highlight_control(ui->locut_cutoff_knob, highlight); + highlight_control(ui->locut_cutoff_knob_2, highlight); + }); +} + +void MainWindow::highlight_limiter_threshold(bool highlight) +{ + post_to_main_thread([this, highlight]{ + highlight_control(ui->limiter_threshold_knob, highlight); + highlight_control(ui->limiter_threshold_knob_2, highlight); + }); +} + +void MainWindow::highlight_makeup_gain(bool highlight) +{ + post_to_main_thread([this, highlight]{ + highlight_control(ui->makeup_gain_knob, highlight); + highlight_control(ui->makeup_gain_knob_2, highlight); + }); +} + +void MainWindow::highlight_stereo_width(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::stereo_width_knob, highlight); +} + +void MainWindow::highlight_treble(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::treble_knob, highlight); +} + +void MainWindow::highlight_mid(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::mid_knob, highlight); +} + +void MainWindow::highlight_bass(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::bass_knob, highlight); +} + +void MainWindow::highlight_gain(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::gainstaging_knob, highlight); +} + +void MainWindow::highlight_compressor_threshold(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::compressor_threshold_knob, highlight); +} + +void MainWindow::highlight_fader(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::fader, highlight); +} + +void MainWindow::highlight_mute(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::mute_button, highlight, /*is_mute_btton=*/true); +} + +void MainWindow::highlight_toggle_locut(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::locut_enabled, highlight); +} + +void MainWindow::highlight_toggle_auto_gain_staging(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::gainstaging_auto_checkbox, highlight); +} + +void MainWindow::highlight_toggle_compressor(unsigned bus_idx, bool highlight) +{ + highlight_control_if_exists(bus_idx, &Ui::AudioExpandedView::compressor_enabled, highlight); +} + +void MainWindow::highlight_toggle_limiter(bool highlight) +{ + post_to_main_thread([this, highlight]{ + highlight_control(ui->limiter_enabled, highlight); + highlight_control(ui->limiter_enabled_2, highlight); + }); +} + +void MainWindow::highlight_toggle_auto_makeup_gain(bool highlight) +{ + post_to_main_thread([this, highlight]{ + highlight_control(ui->makeup_gain_auto_checkbox, highlight); + highlight_control(ui->makeup_gain_auto_checkbox_2, highlight); + }); +} + +template +void MainWindow::set_relative_value(T *control, float value) +{ + post_to_main_thread([control, value]{ + control->setValue(lrintf(control->minimum() + value * (control->maximum() - control->minimum()))); + }); +} + +template +void MainWindow::set_relative_value_if_exists(unsigned bus_idx, T *(Ui_AudioExpandedView::*control), float value) +{ + if (global_audio_mixer != nullptr && + global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::MULTICHANNEL && + bus_idx < audio_expanded_views.size()) { + set_relative_value(audio_expanded_views[bus_idx]->*control, value); + } +} + +template +void MainWindow::click_button_if_exists(unsigned bus_idx, T *(Ui_AudioExpandedView::*control)) +{ + post_to_main_thread([this, bus_idx, control]{ + if (global_audio_mixer != nullptr && + global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::MULTICHANNEL && + bus_idx < audio_expanded_views.size()) { + (audio_expanded_views[bus_idx]->*control)->click(); + } + }); +} + +template +void MainWindow::highlight_control(T *control, bool highlight) +{ + if (control == nullptr) { + return; + } + if (global_audio_mixer == nullptr || + global_audio_mixer->get_mapping_mode() != AudioMixer::MappingMode::MULTICHANNEL) { + highlight = false; + } + if (highlight) { + control->setStyleSheet("background: rgb(0,255,0,80)"); + } else { + control->setStyleSheet(""); + } +} + +template +void MainWindow::highlight_mute_control(T *control, bool highlight) +{ + if (control == nullptr) { + return; + } + if (global_audio_mixer == nullptr || + global_audio_mixer->get_mapping_mode() != AudioMixer::MappingMode::MULTICHANNEL) { + highlight = false; + } + if (highlight) { + control->setStyleSheet("QPushButton { background: rgb(0,255,0,80); } QPushButton:checked { background: rgba(255,80,0,140); }"); + } else { + control->setStyleSheet("QPushButton:checked { background: rgba(255,0,0,80); }"); + } +} + +template +void MainWindow::highlight_control_if_exists(unsigned bus_idx, T *(Ui_AudioExpandedView::*control), bool highlight, bool is_mute_button) +{ + post_to_main_thread([this, bus_idx, control, highlight, is_mute_button]{ + if (bus_idx < audio_expanded_views.size()) { + if (is_mute_button) { + highlight_mute_control(audio_expanded_views[bus_idx]->*control, highlight); + } else { + highlight_control(audio_expanded_views[bus_idx]->*control, highlight); + } + } + }); +} + +void MainWindow::set_transition_names(vector transition_names) +{ + if (transition_names.size() < 1 || transition_names[0].empty()) { + transition_btn1->setText(QString("")); + } else { + transition_btn1->setText(QString::fromStdString(transition_names[0] + " (J)")); + ui->transition_btn1->setShortcut(QKeySequence("J")); + } + if (transition_names.size() < 2 || transition_names[1].empty()) { + transition_btn2->setText(QString("")); + } else { + transition_btn2->setText(QString::fromStdString(transition_names[1] + " (K)")); + ui->transition_btn2->setShortcut(QKeySequence("K")); + } + if (transition_names.size() < 3 || transition_names[2].empty()) { + transition_btn3->setText(QString("")); + } else { + transition_btn3->setText(QString::fromStdString(transition_names[2] + " (L)")); + ui->transition_btn3->setShortcut(QKeySequence("L")); + } +} + +void MainWindow::update_channel_name(Mixer::Output output, const string &name) +{ + if (output >= Mixer::OUTPUT_INPUT0) { + unsigned channel = output - Mixer::OUTPUT_INPUT0; + previews[channel]->label->setText(name.c_str()); + } + + analyzer->update_channel_name(output, name); +} + +void MainWindow::update_channel_color(Mixer::Output output, const string &color) +{ + if (output >= Mixer::OUTPUT_INPUT0) { + unsigned channel = output - Mixer::OUTPUT_INPUT0; + previews[channel]->frame->setStyleSheet(QString::fromStdString("background-color:" + color)); + } +} + +void MainWindow::transition_clicked(int transition_number) +{ + global_mixer->transition_clicked(transition_number); +} + +void MainWindow::channel_clicked(int channel_number) +{ + if (current_wb_pick_display == channel_number) { + // The picking was already done from eventFilter(), since we don't get + // the mouse pointer here. + } else { + global_mixer->channel_clicked(channel_number); + } +} + +void MainWindow::quick_cut_activated(int channel_number) +{ + if (!global_flags.enable_quick_cut_keys) { + return; + } + global_mixer->channel_clicked(channel_number); + global_mixer->transition_clicked(0); +} + +void MainWindow::wb_button_clicked(int channel_number) +{ + current_wb_pick_display = channel_number; + QApplication::setOverrideCursor(Qt::CrossCursor); +} + +void MainWindow::audio_view_changed(int audio_view) +{ + if (audio_view == current_audio_view) { + return; + } + + if (audio_view == 0) { + // Compact audio view. (1, full audio view, has no video previews.) + for (unsigned i = 0; i < previews.size(); ++i) { + QWidget *display = static_cast(previews[i]->frame->parent()); + ui->preview_displays->insertWidget(i, display, 1); + } + } else if (audio_view == 2) { + // Video grid display. + for (unsigned i = 0; i < previews.size(); ++i) { + QWidget *display = static_cast(previews[i]->frame->parent()); + display->setParent(ui->preview_displays_grid); + display->show(); + } + } + + current_audio_view = audio_view; + + // Ask for a relayout, but only after the event loop is done doing relayout + // on everything else. + QMetaObject::invokeMethod(this, "relayout", Qt::QueuedConnection); +} + +bool MainWindow::eventFilter(QObject *watched, QEvent *event) +{ + if (current_wb_pick_display != -1 && + event->type() == QEvent::MouseButtonRelease && + watched->isWidgetType()) { + QApplication::restoreOverrideCursor(); + if (watched == previews[current_wb_pick_display]->display) { + const QMouseEvent *mouse_event = (QMouseEvent *)event; + previews[current_wb_pick_display]->display->grab_white_balance( + current_wb_pick_display, + mouse_event->x(), mouse_event->y()); + } else { + // The user clicked on something else, give up. + // (The click goes through, which might not be ideal, but, yes.) + current_wb_pick_display = -1; + } + } + return false; +} + +void MainWindow::closeEvent(QCloseEvent *event) +{ + if (global_mixer->get_num_connected_clients() > 0) { + QMessageBox::StandardButton reply = + QMessageBox::question(this, "Nageru", "There are clients connected. Do you really want to quit?", + QMessageBox::Yes | QMessageBox::No); + if (reply != QMessageBox::Yes) { + event->ignore(); + return; + } + } + + analyzer->hide(); + event->accept(); +} + +void MainWindow::audio_state_changed() +{ + post_to_main_thread([this]{ + if (global_audio_mixer->get_mapping_mode() == AudioMixer::MappingMode::SIMPLE) { + return; + } + InputMapping mapping = global_audio_mixer->get_input_mapping(); + for (unsigned bus_index = 0; bus_index < mapping.buses.size(); ++bus_index) { + string label = get_bus_desc_label(mapping.buses[bus_index]); + audio_miniviews[bus_index]->bus_desc_label->setFullText( + QString::fromStdString(label)); + audio_expanded_views[bus_index]->bus_desc_label->setFullText( + QString::fromStdString(label)); + } + }); +} diff --git a/nageru/mainwindow.h b/nageru/mainwindow.h new file mode 100644 index 0000000..36be4b8 --- /dev/null +++ b/nageru/mainwindow.h @@ -0,0 +1,179 @@ +#ifndef MAINWINDOW_H +#define MAINWINDOW_H + +#include +#include +#include +#include +#include +#include +#include + +#include "analyzer.h" +#include "audio_mixer.h" +#include "midi_mapper.h" +#include "mixer.h" + +class QEvent; +class QObject; +class QResizeEvent; +class Ui_AudioExpandedView; + +namespace Ui { +class AudioExpandedView; +class AudioMiniView; +class Display; +class MainWindow; +} // namespace Ui + +class QLabel; +class QPushButton; + +class MainWindow : public QMainWindow, public ControllerReceiver +{ + Q_OBJECT + +public: + MainWindow(); + void resizeEvent(QResizeEvent *event) override; + void mixer_created(Mixer *mixer); + + // Used to release FBOs on the global ResourcePool. Call after the + // mixer has been shut down but not destroyed yet. + void mixer_shutting_down(); + +public slots: + void cut_triggered(); + void x264_bitrate_triggered(); + void exit_triggered(); + void manual_triggered(); + void about_triggered(); + void open_analyzer_triggered(); + void simple_audio_mode_triggered(); + void multichannel_audio_mode_triggered(); + void input_mapping_triggered(); + void midi_mapping_triggered(); + void timecode_stream_triggered(); + void timecode_stdout_triggered(); + void transition_clicked(int transition_number); + void channel_clicked(int channel_number); + void quick_cut_activated(int channel_number); + void wb_button_clicked(int channel_number); + void audio_view_changed(int audio_view); + void set_transition_names(std::vector transition_names); + void update_channel_name(Mixer::Output output, const std::string &name); + void update_channel_color(Mixer::Output output, const std::string &color); + void gain_staging_knob_changed(unsigned bus_index, int value); + void final_makeup_gain_knob_changed(int value); + void cutoff_knob_changed(int value); + void stereo_width_knob_changed(unsigned bus_index, int value); + void eq_knob_changed(unsigned bus_index, EQBand band, int value); + void limiter_threshold_knob_changed(int value); + void compressor_threshold_knob_changed(unsigned bus_index, int value); + void mini_fader_changed(int bus, double db_volume); + void mute_button_toggled(int bus, bool checked); + void reset_meters_button_clicked(); + void relayout(); + + // ControllerReceiver interface. + void set_locut(float value) override; + void set_limiter_threshold(float value) override; + void set_makeup_gain(float value) override; + + void set_stereo_width(unsigned bus_idx, float value) override; + void set_treble(unsigned bus_idx, float value) override; + void set_mid(unsigned bus_idx, float value) override; + void set_bass(unsigned bus_idx, float value) override; + void set_gain(unsigned bus_idx, float value) override; + void set_compressor_threshold(unsigned bus_idx, float value) override; + void set_fader(unsigned bus_idx, float value) override; + + void toggle_mute(unsigned bus_idx) override; + void toggle_locut(unsigned bus_idx) override; + void toggle_auto_gain_staging(unsigned bus_idx) override; + void toggle_compressor(unsigned bus_idx) override; + void clear_peak(unsigned bus_idx) override; + void toggle_limiter() override; + void toggle_auto_makeup_gain() override; + + void clear_all_highlights() override; + + void highlight_locut(bool highlight) override; + void highlight_limiter_threshold(bool highlight) override; + void highlight_makeup_gain(bool highlight) override; + + void highlight_stereo_width(unsigned bus_idx, bool highlight) override; + void highlight_treble(unsigned bus_idx, bool highlight) override; + void highlight_mid(unsigned bus_idx, bool highlight) override; + void highlight_bass(unsigned bus_idx, bool highlight) override; + void highlight_gain(unsigned bus_idx, bool highlight) override; + void highlight_compressor_threshold(unsigned bus_idx, bool highlight) override; + void highlight_fader(unsigned bus_idx, bool highlight) override; + + void highlight_mute(unsigned bus_idx, bool highlight) override; + void highlight_toggle_locut(unsigned bus_idx, bool highlight) override; + void highlight_toggle_auto_gain_staging(unsigned bus_idx, bool highlight) override; + void highlight_toggle_compressor(unsigned bus_idx, bool highlight) override; + void highlight_clear_peak(unsigned bus_idx, bool highlight) override {} // We don't mark this currently. + void highlight_toggle_limiter(bool highlight) override; + void highlight_toggle_auto_makeup_gain(bool highlight) override; + + // Raw receivers are not used. + void controller_changed(unsigned controller) override {} + void note_on(unsigned note) override {} + +private: + void reset_audio_mapping_ui(); + void setup_audio_miniview(); + void setup_audio_expanded_view(); + bool eventFilter(QObject *watched, QEvent *event) override; + void closeEvent(QCloseEvent *event) override; + void update_cutoff_labels(float cutoff_hz); + void update_stereo_knob_and_label(unsigned bus_index, int stereo_width_percent); + void update_stereo_label(unsigned bus_index, int stereo_width_percent); + void update_eq_label(unsigned bus_index, EQBand band, float gain_db); + void setup_theme_menu(); + + // Called from DiskSpaceEstimator. + void report_disk_space(off_t free_bytes, double estimated_seconds_left); + + // Called from the mixer. + void audio_level_callback(float level_lufs, float peak_db, std::vector bus_levels, float global_level_lufs, float range_low_lufs, float range_high_lufs, float final_makeup_gain_db, float correlation); + std::chrono::steady_clock::time_point last_audio_level_callback; + + void audio_state_changed(); + + template + void set_relative_value(T *control, float value); + + template + void set_relative_value_if_exists(unsigned bus_idx, T *Ui_AudioExpandedView::*control, float value); + + template + void click_button_if_exists(unsigned bus_idx, T *Ui_AudioExpandedView::*control); + + template + void highlight_control(T *control, bool highlight); + + template + void highlight_mute_control(T *control, bool highlight); + + template + void highlight_control_if_exists(unsigned bus_idx, T *(Ui_AudioExpandedView::*control), bool highlight, bool is_mute_button = false); + + Ui::MainWindow *ui; + QLabel *disk_free_label; + QMenu *theme_menu = nullptr; + QPushButton *transition_btn1, *transition_btn2, *transition_btn3; + std::vector previews; + std::vector audio_miniviews; + std::vector audio_expanded_views; + int current_wb_pick_display = -1; + int current_audio_view = -1; + MIDIMapper midi_mapper; + std::unique_ptr analyzer; +}; + +extern MainWindow *global_mainwindow; + +#endif diff --git a/nageru/mainwindow.ui b/nageru/mainwindow.ui new file mode 100644 index 0000000..d727155 --- /dev/null +++ b/nageru/mainwindow.ui @@ -0,0 +1,1643 @@ + + + MainWindow + + + + 0 + 0 + 1089 + 664 + + + + Nageru + + + + true + + + + 0 + 0 + + + + + + + + + + + 0 + + + + + + 1 + 1 + + + + + + + + Preview + + + Qt::AlignCenter + + + + + + + + + 25 + + + 0 + + + 20 + + + + + + 0 + 0 + + + + + 115 + 0 + + + + + 16777215 + 16777215 + + + + Cut + + + + + + + + 0 + 0 + + + + + 115 + 0 + + + + + 16777215 + 16777215 + + + + Fade + + + + + + + + 0 + 0 + + + + + 115 + 0 + + + + + 16777215 + 16777215 + + + + Wipe + + + + + + + + + 0 + + + + + + 1 + 1 + + + + + 16 + 9 + + + + + 16 + 9 + + + + + + + + Live + + + Qt::AlignCenter + + + + + + + + + 0 + + + + + + 0 + 0 + + + + + 0 + 14 + + + + + + + + + 255 + 255 + 255 + + + + + + + 239 + 0 + 4 + + + + + + + + + 255 + 255 + 255 + + + + + + + 239 + 0 + 4 + + + + + + + + + 239 + 0 + 4 + + + + + + + 239 + 0 + 4 + + + + + + + + true + + + + + + + + + 0 + + + 4 + + + + + 0 + + + + + + 0 + 1 + + + + + 16 + 0 + + + + + 1 + 0 + + + + + 0 + 0 + + + + + + + + + 255 + 255 + 255 + + + + + + + 5 + 239 + 111 + + + + + + + + + 255 + 255 + 255 + + + + + + + 5 + 239 + 111 + + + + + + + + + 5 + 239 + 111 + + + + + + + 5 + 239 + 111 + + + + + + + + true + + + + + + + + + + 30 + 0 + + + + -0.0 + + + Qt::AlignCenter + + + + + + + + + 3 + + + 0 + + + + + + + + 0 + 0 + + + + + 24 + 0 + + + + + + + + + 255 + 255 + 255 + + + + + + + 0 + 239 + 219 + + + + + + + + + 255 + 255 + 255 + + + + + + + 0 + 239 + 219 + + + + + + + + + 0 + 239 + 219 + + + + + + + 0 + 239 + 219 + + + + + + + + true + + + + + + + + + + 30 + 20 + + + + RST + + + false + + + + + + + + + + + + + + + 0 + + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + 8 + 0 + + + + + 0 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + Compact audio view (1/3) + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + true + + + + 15 + 15 + + + + ... + + + true + + + Qt::LeftArrow + + + + + + + + 15 + 15 + + + + ... + + + true + + + Qt::RightArrow + + + + + + + + + + 6 + + + 0 + + + + + QFrame::NoFrame + + + Qt::ScrollBarAlwaysOff + + + true + + + + + 0 + 0 + 497 + 235 + + + + + 0 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + QLayout::SetFixedSize + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + 0 + + + + + + 64 + 64 + + + + + 16777215 + 64 + + + + -400 + + + 0 + + + -260 + + + 30.000000000000000 + + + true + + + + + + + Auto + + + true + + + + + + + Compr. threshold + + + + + + + Gain staging + + + Qt::AlignCenter + + + + + + + -26.0 dB + + + Qt::AlignCenter + + + + + + + + 64 + 64 + + + + + 16777215 + 64 + + + + -300 + + + 300 + + + 60.000000000000000 + + + true + + + + + + + -0.0 dB + + + Qt::AlignCenter + + + + + + + -14.0 dB + + + Qt::AlignCenter + + + + + + + Enabled + + + true + + + + + + + Auto + + + true + + + + + + + + 64 + 64 + + + + + 16777215 + 64 + + + + -400 + + + 0 + + + -140 + + + 30.000000000000000 + + + true + + + + + + + 120 Hz + + + Qt::AlignCenter + + + + + + + Enabled + + + true + + + + + + + -0.0 dB + + + Qt::AlignCenter + + + + + + + Enabled + + + true + + + + + + + Lo-cut (24dB/oct) + + + + + + + Qt::Vertical + + + QSizePolicy::Expanding + + + + 20 + 40 + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + 64 + 64 + + + + + 16777215 + 64 + + + + -150 + + + 150 + + + 30.000000000000000 + + + true + + + + + + + Limiter threshold + + + Qt::AlignCenter + + + + + + + Makeup gain + + + Qt::AlignCenter + + + + + + + + 0 + 0 + + + + + 64 + 64 + + + + + 16777215 + 64 + + + + 60 + + + 26 + + + + + + + + + + + + + Qt::Horizontal + + + QSizePolicy::Preferred + + + + 0 + 40 + + + + + + + + + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + 0 + + + + + Full audio view (2/3) + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + true + + + + 15 + 15 + + + + ... + + + true + + + Qt::LeftArrow + + + + + + + + 15 + 15 + + + + ... + + + true + + + Qt::RightArrow + + + + + + + + + 6 + + + 0 + + + + + QFrame::NoFrame + + + Qt::ScrollBarAlwaysOff + + + true + + + + + 0 + 0 + 722 + 281 + + + + + 0 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + QLayout::SetFixedSize + + + + + + + Qt::Horizontal + + + + 723 + 20 + + + + + + + + + + + + 0 + + + + + 120 Hz + + + Qt::AlignCenter + + + + + + + -0.0 dB + + + Qt::AlignCenter + + + + + + + -14.0 dB + + + Qt::AlignCenter + + + + + + + + 0 + 0 + + + + + 64 + 64 + + + + + 16777215 + 64 + + + + 60 + + + 26 + + + + + + + Enabled + + + true + + + + + + + + 64 + 64 + + + + + 16777215 + 64 + + + + -400 + + + 0 + + + -140 + + + 30.000000000000000 + + + true + + + + + + + Auto + + + true + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Lo-cut (24dB/oct) + + + + + + + Qt::Vertical + + + QSizePolicy::Expanding + + + + 20 + 40 + + + + + + + + + 64 + 64 + + + + + 16777215 + 64 + + + + -150 + + + 150 + + + 30.000000000000000 + + + true + + + + + + + Limiter threshold + + + Qt::AlignCenter + + + + + + + Makeup gain + + + Qt::AlignCenter + + + + + + + + + + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + 0 + + + + + Video grid display (3/3) + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + true + + + + 15 + 15 + + + + ... + + + true + + + Qt::LeftArrow + + + + + + + + 15 + 15 + + + + ... + + + true + + + Qt::RightArrow + + + + + + + + + + + + + + + + + + + + 0 + 0 + 1089 + 22 + + + + + &Video + + + + Display &time code + + + + + + + HDMI/SDI output device + + + + + + HDMI/SDI output resolution + + + + + + + + + + + + + + + &Help + + + + + + + &Audio + + + + + + + + + + + + + + &Exit + + + + + &Begin new video segment + + + + + &About Nageru… + + + + + Change &x264 bitrate… + + + + + &Input mapping… + + + + + true + + + true + + + Simple + + + + + true + + + Multichannel + + + + + Setup MIDI controller… + + + + + Online &manual… + + + + + true + + + In &stream + + + + + true + + + On standard &output + + + + + Open frame &analyzer… + + + + + true + + + Enable &quick-cut keys (Q, W, E, etc.) + + + + + + + VUMeter + QWidget +
vumeter.h
+ 1 + + + ClickableLabel + QLabel +
clickable_label.h
+ + + GLWidget + QWidget +
glwidget.h
+ + + LRAMeter + QWidget +
lrameter.h
+ 1 + + + CorrelationMeter + QWidget +
correlation_meter.h
+ 1 + + + + + diff --git a/nageru/memcpy_interleaved.cpp b/nageru/memcpy_interleaved.cpp new file mode 100644 index 0000000..9a41cdd --- /dev/null +++ b/nageru/memcpy_interleaved.cpp @@ -0,0 +1,136 @@ +#include +#include +#include +#if __SSE2__ +#include +#endif + +using namespace std; + +// TODO: Support stride. +void memcpy_interleaved_slow(uint8_t *dest1, uint8_t *dest2, const uint8_t *src, size_t n) +{ + assert(n % 2 == 0); + uint8_t *dptr1 = dest1; + uint8_t *dptr2 = dest2; + + for (size_t i = 0; i < n; i += 2) { + *dptr1++ = *src++; + *dptr2++ = *src++; + } +} + +#ifdef __SSE2__ + +// Returns the number of bytes consumed. +size_t memcpy_interleaved_fastpath(uint8_t *dest1, uint8_t *dest2, const uint8_t *src, size_t n) +{ + const uint8_t *limit = src + n; + size_t consumed = 0; + + // Align end to 32 bytes. + limit = (const uint8_t *)(intptr_t(limit) & ~31); + + if (src >= limit) { + return 0; + } + + // Process [0,31] bytes, such that start gets aligned to 32 bytes. + const uint8_t *aligned_src = (const uint8_t *)(intptr_t(src + 31) & ~31); + if (aligned_src != src) { + size_t n2 = aligned_src - src; + memcpy_interleaved_slow(dest1, dest2, src, n2); + dest1 += n2 / 2; + dest2 += n2 / 2; + if (n2 % 2) { + swap(dest1, dest2); + } + src = aligned_src; + consumed += n2; + } + + // Make the length a multiple of 64. + if (((limit - src) % 64) != 0) { + limit -= 32; + } + assert(((limit - src) % 64) == 0); + +#if __AVX2__ + const __m256i * __restrict in = (const __m256i *)src; + __m256i * __restrict out1 = (__m256i *)dest1; + __m256i * __restrict out2 = (__m256i *)dest2; + + __m256i shuffle_cw = _mm256_set_epi8( + 15, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 0, + 15, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 0); + while (in < (const __m256i *)limit) { + // Note: For brevity, comments show lanes as if they were 2x64-bit (they're actually 2x128). + __m256i data1 = _mm256_stream_load_si256(in); // AaBbCcDd EeFfGgHh + __m256i data2 = _mm256_stream_load_si256(in + 1); // IiJjKkLl MmNnOoPp + + data1 = _mm256_shuffle_epi8(data1, shuffle_cw); // ABCDabcd EFGHefgh + data2 = _mm256_shuffle_epi8(data2, shuffle_cw); // IJKLijkl MNOPmnop + + data1 = _mm256_permute4x64_epi64(data1, 0b11011000); // ABCDEFGH abcdefgh + data2 = _mm256_permute4x64_epi64(data2, 0b11011000); // IJKLMNOP ijklmnop + + __m256i lo = _mm256_permute2x128_si256(data1, data2, 0b00100000); + __m256i hi = _mm256_permute2x128_si256(data1, data2, 0b00110001); + + _mm256_storeu_si256(out1, lo); + _mm256_storeu_si256(out2, hi); + + in += 2; + ++out1; + ++out2; + consumed += 64; + } +#else + const __m128i * __restrict in = (const __m128i *)src; + __m128i * __restrict out1 = (__m128i *)dest1; + __m128i * __restrict out2 = (__m128i *)dest2; + + __m128i mask_lower_byte = _mm_set1_epi16(0x00ff); + while (in < (const __m128i *)limit) { + __m128i data1 = _mm_load_si128(in); + __m128i data2 = _mm_load_si128(in + 1); + __m128i data1_lo = _mm_and_si128(data1, mask_lower_byte); + __m128i data2_lo = _mm_and_si128(data2, mask_lower_byte); + __m128i data1_hi = _mm_srli_epi16(data1, 8); + __m128i data2_hi = _mm_srli_epi16(data2, 8); + __m128i lo = _mm_packus_epi16(data1_lo, data2_lo); + _mm_storeu_si128(out1, lo); + __m128i hi = _mm_packus_epi16(data1_hi, data2_hi); + _mm_storeu_si128(out2, hi); + + in += 2; + ++out1; + ++out2; + consumed += 32; + } +#endif + + return consumed; +} + +#endif // defined(__SSE2__) + +void memcpy_interleaved(uint8_t *dest1, uint8_t *dest2, const uint8_t *src, size_t n) +{ +#ifdef __SSE2__ + size_t consumed = memcpy_interleaved_fastpath(dest1, dest2, src, n); + src += consumed; + dest1 += consumed / 2; + dest2 += consumed / 2; + if (consumed % 2) { + swap(dest1, dest2); + } + n -= consumed; + + if (n > 0) { + memcpy_interleaved_slow(dest1, dest2, src, n); + } +#else + memcpy_interleaved_slow(dest1, dest2, src, n); +#endif +} diff --git a/nageru/memcpy_interleaved.h b/nageru/memcpy_interleaved.h new file mode 100644 index 0000000..a7f8994 --- /dev/null +++ b/nageru/memcpy_interleaved.h @@ -0,0 +1,11 @@ +#ifndef _MEMCPY_INTERLEAVED_H +#define _MEMCPY_INTERLEAVED_H 1 + +#include +#include + +// Copies every other byte from src to dest1 and dest2. +// TODO: Support stride. +void memcpy_interleaved(uint8_t *dest1, uint8_t *dest2, const uint8_t *src, size_t n); + +#endif // !defined(_MEMCPY_INTERLEAVED_H) diff --git a/nageru/meson.build b/nageru/meson.build new file mode 100644 index 0000000..70d5ab7 --- /dev/null +++ b/nageru/meson.build @@ -0,0 +1,244 @@ +qt5 = import('qt5') +protoc = find_program('protoc') +cxx = meson.get_compiler('cpp') + +# Use lld if we can; it links a lot faster than ld.bfd or gold. +nageru_link_args = [] +code = '''#include +int main() { printf("Hello, world!\n"); return 0; } +''' +if cxx.links(code, args: '-fuse-ld=lld', name: 'check for LLD') + nageru_link_args += '-fuse-ld=lld' +endif + +embedded_bmusb = get_option('embedded_bmusb') + +alsadep = dependency('alsa') +bmusbdep = dependency('bmusb', required: not embedded_bmusb) +dldep = cxx.find_library('dl') +epoxydep = dependency('epoxy') +libavcodecdep = dependency('libavcodec') +libavformatdep = dependency('libavformat') +libavresampledep = dependency('libavresample') +libavutildep = dependency('libavutil') +libjpegdep = dependency('libjpeg') +libmicrohttpddep = dependency('libmicrohttpd') +libswscaledep = dependency('libswscale') +libusbdep = dependency('libusb-1.0') +luajitdep = dependency('luajit') +movitdep = dependency('movit') +protobufdep = dependency('protobuf') +qcustomplotdep = cxx.find_library('qcustomplot') +qt5deps = dependency('qt5', modules: ['Core', 'Gui', 'Widgets', 'OpenGLExtensions', 'OpenGL', 'PrintSupport']) +threaddep = dependency('threads') +vadrmdep = dependency('libva-drm') +vax11dep = dependency('libva-x11') +x11dep = dependency('x11') +x264dep = dependency('x264') +zitaresamplerdep = cxx.find_library('zita-resampler') + +srcs = [] +nageru_deps = [qt5deps, libjpegdep, movitdep, libmicrohttpddep, protobufdep, + vax11dep, vadrmdep, x11dep, libavformatdep, libavresampledep, libavcodecdep, libavutildep, + libswscaledep, libusbdep, luajitdep, dldep, x264dep, alsadep, zitaresamplerdep, + qcustomplotdep, threaddep] +nageru_include_dirs = [] +nageru_link_with = [] +nageru_build_rpath = '' +nageru_install_rpath = '' + +kaeru_link_with = [] +kaeru_extra_deps = [] + +# DeckLink has these issues, and we include it from various places. +if cxx.has_argument('-Wno-non-virtual-dtor') + add_project_arguments('-Wno-non-virtual-dtor', language: 'cpp') +endif + +# FFmpeg has a lot of deprecated APIs whose replacements are not available +# in Debian stable, so we suppress these warnings. +if cxx.has_argument('-Wno-deprecated-declarations') + add_project_arguments('-Wno-deprecated-declarations', language: 'cpp') +endif + +# CEF. +exe_dir = join_paths(get_option('prefix'), 'lib/nageru') +cef_dir = get_option('cef_dir') +cef_build_type = get_option('cef_build_type') +have_cef = (cef_dir != '') +if have_cef + add_project_arguments('-DHAVE_CEF=1', language: 'cpp') + + system_cef = (cef_build_type == 'system') + if system_cef + cef_lib_dir = cef_dir + cef_resource_dir = '/usr/share/cef/Resources' + else + cef_lib_dir = join_paths(cef_dir, cef_build_type) + cef_resource_dir = join_paths(cef_dir, 'Resources') + + nageru_include_dirs += include_directories(cef_dir) + nageru_include_dirs += include_directories(join_paths(cef_dir, 'include')) + nageru_build_rpath = cef_lib_dir + nageru_install_rpath = '$ORIGIN/' + endif + + cefdep = cxx.find_library('cef') + nageru_deps += cefdep + + # CEF wrapper library; not built as part of the CEF binary distribution, + # but should be if CEF is installed as a system library. + if system_cef + cefdlldep = cxx.find_library('cef_dll_wrapper') + nageru_deps += cefdlldep + else + cmake = find_program('cmake') + cef_compile_script = find_program('scripts/compile_cef_dll_wrapper.sh') + + cef_dll_target = custom_target('libcef_dll_wrapper', + input: join_paths(cef_dir, 'libcef_dll/CMakeLists.txt'), + output: ['libcef_dll_wrapper.a', 'cef-stamp'], + command: [cef_compile_script, '@BUILD_DIR@', cef_dir, cmake, '@OUTPUT@']) + + # Putting the .a in sources seemingly hits a bug where the .a files get sorted + # in the wrong order. This is a workaround; see + # https://github.com/mesonbuild/meson/issues/3613#issuecomment-408276296 . + cefdlldep = declare_dependency(sources: cef_dll_target[1], link_args: cef_dll_target.full_path()) + nageru_deps += cefdlldep + endif + + cef_libs = ['libEGL.so', 'libGLESv2.so', 'natives_blob.bin', 'snapshot_blob.bin', 'v8_context_snapshot.bin'] + cef_resources = ['cef.pak', 'cef_100_percent.pak', 'cef_200_percent.pak', 'cef_extensions.pak', 'devtools_resources.pak'] + if not get_option('cef_no_icudtl') + cef_resources += ['icudtl.dat'] + endif + if cef_build_type != 'system' + cef_libs += ['libcef.so'] + endif + + # Symlink the files into the build directory, so that running nageru without ninja install works. + run_command('mkdir', join_paths(meson.current_build_dir(), 'locales/')) + foreach file : cef_libs + run_command('ln', '-s', join_paths(cef_lib_dir, file), meson.current_build_dir()) + install_data(join_paths(cef_lib_dir, file), install_dir: exe_dir) + endforeach + foreach file : cef_resources + run_command('ln', '-s', join_paths(cef_resource_dir, file), meson.current_build_dir()) + install_data(join_paths(cef_resource_dir, file), install_dir: exe_dir) + endforeach + run_command('ln', '-s', join_paths(cef_resource_dir, 'locales/en-US.pak'), join_paths(meson.current_build_dir(), 'locales/')) + install_data(join_paths(cef_resource_dir, 'locales/en-US.pak'), install_dir: join_paths(exe_dir, 'locales')) +endif + +# bmusb. +if embedded_bmusb + bmusb_dir = include_directories('bmusb') + nageru_include_dirs += bmusb_dir + + bmusb = static_library('bmusb', 'bmusb/bmusb.cpp', 'bmusb/fake_capture.cpp', + dependencies: [libusbdep], + include_directories: [bmusb_dir]) + nageru_link_with += bmusb + kaeru_link_with += bmusb +else + nageru_deps += bmusbdep + kaeru_extra_deps += bmusbdep +endif + +# Protobuf compilation. +gen = generator(protoc, \ + output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], + arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@']) +proto_generated = gen.process(['state.proto', 'midi_mapping.proto', 'json.proto']) +protobuf_lib = static_library('protobufs', proto_generated, dependencies: nageru_deps, include_directories: nageru_include_dirs) +protobuf_hdrs = declare_dependency(sources: proto_generated) +nageru_link_with += protobuf_lib + +# Preprocess Qt as needed. +qt_files = qt5.preprocess( + moc_headers: ['aboutdialog.h', 'analyzer.h', 'clickable_label.h', 'compression_reduction_meter.h', 'correlation_meter.h', + 'ellipsis_label.h', 'glwidget.h', 'input_mapping_dialog.h', 'lrameter.h', 'mainwindow.h', 'midi_mapping_dialog.h', + 'nonlinear_fader.h', 'vumeter.h'], + ui_files: ['aboutdialog.ui', 'analyzer.ui', 'audio_expanded_view.ui', 'audio_miniview.ui', 'display.ui', + 'input_mapping.ui', 'mainwindow.ui', 'midi_mapping.ui'], + dependencies: qt5deps) + +# Qt objects. +srcs += ['glwidget.cpp', 'mainwindow.cpp', 'vumeter.cpp', 'lrameter.cpp', 'compression_reduction_meter.cpp', + 'correlation_meter.cpp', 'aboutdialog.cpp', 'analyzer.cpp', 'input_mapping_dialog.cpp', 'midi_mapping_dialog.cpp', + 'nonlinear_fader.cpp', 'context_menus.cpp', 'vu_common.cpp', 'piecewise_interpolator.cpp', 'midi_mapper.cpp'] + +# Auxiliary objects used for nearly everything. +aux_srcs = ['metrics.cpp', 'flags.cpp'] +aux = static_library('aux', aux_srcs, dependencies: nageru_deps, include_directories: nageru_include_dirs) +nageru_link_with += aux + +# Audio objects. +audio_mixer_srcs = ['audio_mixer.cpp', 'alsa_input.cpp', 'alsa_pool.cpp', 'ebu_r128_proc.cc', 'stereocompressor.cpp', + 'resampling_queue.cpp', 'flags.cpp', 'correlation_measurer.cpp', 'filter.cpp', 'input_mapping.cpp'] +audio = static_library('audio', audio_mixer_srcs, dependencies: [nageru_deps, protobuf_hdrs], include_directories: nageru_include_dirs) +nageru_link_with += audio + +# Mixer objects. +srcs += ['chroma_subsampler.cpp', 'v210_converter.cpp', 'mixer.cpp', 'pbo_frame_allocator.cpp', + 'context.cpp', 'theme.cpp', 'image_input.cpp', 'alsa_output.cpp', + 'disk_space_estimator.cpp', 'timecode_renderer.cpp', 'tweaked_inputs.cpp'] + +# Streaming and encoding objects (largely the set that is shared between Nageru and Kaeru). +stream_srcs = ['quicksync_encoder.cpp', 'x264_encoder.cpp', 'x264_dynamic.cpp', 'x264_speed_control.cpp', 'video_encoder.cpp', + 'metacube2.cpp', 'mux.cpp', 'audio_encoder.cpp', 'ffmpeg_raii.cpp', 'ffmpeg_util.cpp', 'httpd.cpp', 'ffmpeg_capture.cpp', + 'print_latency.cpp', 'basic_stats.cpp', 'ref_counted_frame.cpp'] +stream = static_library('stream', stream_srcs, dependencies: nageru_deps, include_directories: nageru_include_dirs) +nageru_link_with += stream + +# DeckLink. +srcs += ['decklink_capture.cpp', 'decklink_util.cpp', 'decklink_output.cpp', 'memcpy_interleaved.cpp', + 'decklink/DeckLinkAPIDispatch.cpp'] +decklink_dir = include_directories('decklink') +nageru_include_dirs += decklink_dir + +# CEF input. +if have_cef + srcs += ['nageru_cef_app.cpp', 'cef_capture.cpp'] +endif + +srcs += qt_files +srcs += proto_generated + +# Everything except main.cpp. (We do this because if you specify a .cpp file in +# both Nageru and Kaeru, it gets compiled twice. In the older Makefiles, Kaeru +# depended on a smaller set of objects.) +core = static_library('core', srcs, dependencies: nageru_deps, include_directories: nageru_include_dirs) +nageru_link_with += core + +# Nageru executable; it goes into /usr/lib/nageru since CEF files go there, too +# (we can't put them straight into /usr/bin). +executable('nageru', 'main.cpp', + dependencies: nageru_deps, + include_directories: nageru_include_dirs, + link_with: nageru_link_with, + link_args: nageru_link_args, + build_rpath: nageru_build_rpath, + install_rpath: nageru_install_rpath, + install: true, + install_dir: exe_dir +) +meson.add_install_script('scripts/setup_nageru_symlink.sh') + +# Kaeru executable. +executable('kaeru', 'kaeru.cpp', + dependencies: [nageru_deps, kaeru_extra_deps], + include_directories: nageru_include_dirs, + link_with: [stream, aux, kaeru_link_with], + link_args: nageru_link_args, + install: true) + +# Audio mixer microbenchmark. +executable('benchmark_audio_mixer', 'benchmark_audio_mixer.cpp', dependencies: nageru_deps, include_directories: nageru_include_dirs, link_args: nageru_link_args, link_with: [audio, aux]) + +# These are needed for a default run. +data_files = ['theme.lua', 'simple.lua', 'bg.jpeg', 'akai_midimix.midimapping'] +install_data(data_files, install_dir: join_paths(get_option('prefix'), 'share/nageru')) +foreach file : data_files + run_command('ln', '-s', join_paths(meson.current_source_dir(), file), meson.current_build_dir()) +endforeach diff --git a/nageru/metacube2.cpp b/nageru/metacube2.cpp new file mode 100644 index 0000000..6b68132 --- /dev/null +++ b/nageru/metacube2.cpp @@ -0,0 +1,60 @@ +/* + * Implementation of Metacube2 utility functions. + * + * Note: This file is meant to compile as both C and C++, for easier inclusion + * in other projects. + */ + +#include "metacube2.h" + +#include +#include + +/* + * https://www.ece.cmu.edu/~koopman/pubs/KoopmanCRCWebinar9May2012.pdf + * recommends this for messages as short as ours (see table at page 34). + */ +#define METACUBE2_CRC_POLYNOMIAL 0x8FDB + +/* Semi-random starting value to make sure all-zero won't pass. */ +#define METACUBE2_CRC_START 0x1234 + +/* This code is based on code generated by pycrc. */ +uint16_t metacube2_compute_crc(const struct metacube2_block_header *hdr) +{ + static const int data_len = sizeof(hdr->size) + sizeof(hdr->flags); + const uint8_t *data = (uint8_t *)&hdr->size; + uint16_t crc = METACUBE2_CRC_START; + int i, j; + + for (i = 0; i < data_len; ++i) { + uint8_t c = data[i]; + for (j = 0; j < 8; j++) { + int bit = crc & 0x8000; + crc = (crc << 1) | ((c >> (7 - j)) & 0x01); + if (bit) { + crc ^= METACUBE2_CRC_POLYNOMIAL; + } + } + } + + /* Finalize. */ + for (i = 0; i < 16; i++) { + int bit = crc & 0x8000; + crc = crc << 1; + if (bit) { + crc ^= METACUBE2_CRC_POLYNOMIAL; + } + } + + /* + * Invert the checksum for metadata packets, so that clients that + * don't understand metadata will ignore it as broken. There will + * probably be logging, but apart from that, it's harmless. + */ + if (ntohs(hdr->flags) & METACUBE_FLAGS_METADATA) { + crc ^= 0xffff; + } + + return crc; +} diff --git a/nageru/metacube2.h b/nageru/metacube2.h new file mode 100644 index 0000000..4f232c8 --- /dev/null +++ b/nageru/metacube2.h @@ -0,0 +1,71 @@ +#ifndef _METACUBE2_H +#define _METACUBE2_H + +/* + * Definitions for the Metacube2 protocol, used to communicate with Cubemap. + * + * Note: This file is meant to compile as both C and C++, for easier inclusion + * in other projects. + */ + +#include + +#define METACUBE2_SYNC "cube!map" /* 8 bytes long. */ +#define METACUBE_FLAGS_HEADER 0x1 +#define METACUBE_FLAGS_NOT_SUITABLE_FOR_STREAM_START 0x2 + +/* + * Metadata packets; should not be counted as data, but rather + * parsed (or ignored if you don't understand them). + * + * Metadata packets start with a uint64_t (network byte order) + * that describe the type; the rest is defined by the type. + */ +#define METACUBE_FLAGS_METADATA 0x4 + +struct metacube2_block_header { + char sync[8]; /* METACUBE2_SYNC */ + uint32_t size; /* Network byte order. Does not include header. */ + uint16_t flags; /* Network byte order. METACUBE_FLAGS_*. */ + uint16_t csum; /* Network byte order. CRC16 of size and flags. + If METACUBE_FLAGS_METADATA is set, inverted + so that older clients will ignore it as broken. */ +}; + +uint16_t metacube2_compute_crc(const struct metacube2_block_header *hdr); + +/* + * Set by the encoder, and can be measured for latency purposes (e.g., if the + * network can't keep up, the latency will tend to increase. + */ +#define METACUBE_METADATA_TYPE_ENCODER_TIMESTAMP 0x1 + +struct metacube2_timestamp_packet { + uint64_t type; /* METACUBE_METADATA_TYPE_ENCODER_TIMESTAMP, in network byte order. */ + + /* + * Time since the UTC epoch. Basically a struct timespec. + * Both are in network byte order. + */ + uint64_t tv_sec; + uint64_t tv_nsec; +}; + +/* + * Sent before a block to mark its presentation timestamp (ie., counts + * only for the next Metacube block). Used so that the reflector can know + * the length (in seconds) of fragments. + */ +#define METACUBE_METADATA_TYPE_NEXT_BLOCK_PTS 0x2 + +struct metacube2_pts_packet { + uint64_t type; /* METACUBE_METADATA_TYPE_NEXT_BLOCK_PTS, in network byte order. */ + + /* The timestamp of the first packet in the next block, in network byte order. */ + int64_t pts; + + /* Timebase "pts" is expressed in, as a fraction. Network byte order. */ + uint64_t timebase_num, timebase_den; +}; + +#endif /* !defined(_METACUBE_H) */ diff --git a/nageru/metrics.cpp b/nageru/metrics.cpp new file mode 100644 index 0000000..86c3d59 --- /dev/null +++ b/nageru/metrics.cpp @@ -0,0 +1,332 @@ +#include "metrics.h" + +#include +#include + +#include +#include +#include +#include + +using namespace std; +using namespace std::chrono; + +Metrics global_metrics; + +double get_timestamp_for_metrics() +{ + return duration(system_clock::now().time_since_epoch()).count(); +} + +string Metrics::serialize_name(const string &name, const vector> &labels) +{ + return "nageru_" + name + serialize_labels(labels); +} + +string Metrics::serialize_labels(const vector> &labels) +{ + if (labels.empty()) { + return ""; + } + + string label_str; + for (const pair &label : labels) { + if (!label_str.empty()) { + label_str += ","; + } + label_str += label.first + "=\"" + label.second + "\""; + } + return "{" + label_str + "}"; +} + +void Metrics::add(const string &name, const vector> &labels, atomic *location, Metrics::Type type) +{ + Metric metric; + metric.data_type = DATA_TYPE_INT64; + metric.location_int64 = location; + + lock_guard lock(mu); + metrics.emplace(MetricKey(name, labels), metric); + assert(types.count(name) == 0 || types[name] == type); + types[name] = type; +} + +void Metrics::add(const string &name, const vector> &labels, atomic *location, Metrics::Type type) +{ + Metric metric; + metric.data_type = DATA_TYPE_DOUBLE; + metric.location_double = location; + + lock_guard lock(mu); + metrics.emplace(MetricKey(name, labels), metric); + assert(types.count(name) == 0 || types[name] == type); + types[name] = type; +} + +void Metrics::add(const string &name, const vector> &labels, Histogram *location, Laziness laziness) +{ + Metric metric; + metric.data_type = DATA_TYPE_HISTOGRAM; + metric.laziness = laziness; + metric.location_histogram = location; + + lock_guard lock(mu); + metrics.emplace(MetricKey(name, labels), metric); + assert(types.count(name) == 0 || types[name] == TYPE_HISTOGRAM); + types[name] = TYPE_HISTOGRAM; +} + +void Metrics::add(const string &name, const vector> &labels, Summary *location, Laziness laziness) +{ + Metric metric; + metric.data_type = DATA_TYPE_SUMMARY; + metric.laziness = laziness; + metric.location_summary = location; + + lock_guard lock(mu); + metrics.emplace(MetricKey(name, labels), metric); + assert(types.count(name) == 0 || types[name] == TYPE_SUMMARY); + types[name] = TYPE_SUMMARY; +} + +void Metrics::remove(const string &name, const vector> &labels) +{ + lock_guard lock(mu); + + auto it = metrics.find(MetricKey(name, labels)); + assert(it != metrics.end()); + + // If this is the last metric with this name, remove the type as well. + if (!((it != metrics.begin() && prev(it)->first.name == name) || + (it != metrics.end() && next(it)->first.name == name))) { + types.erase(name); + } + + metrics.erase(it); +} + +string Metrics::serialize() const +{ + stringstream ss; + ss.imbue(locale("C")); + ss.precision(20); + + lock_guard lock(mu); + auto type_it = types.cbegin(); + for (const auto &key_and_metric : metrics) { + string name = "nageru_" + key_and_metric.first.name + key_and_metric.first.serialized_labels; + const Metric &metric = key_and_metric.second; + + if (type_it != types.cend() && + key_and_metric.first.name == type_it->first) { + // It's the first time we print out any metric with this name, + // so add the type header. + if (type_it->second == TYPE_GAUGE) { + ss << "# TYPE nageru_" << type_it->first << " gauge\n"; + } else if (type_it->second == TYPE_HISTOGRAM) { + ss << "# TYPE nageru_" << type_it->first << " histogram\n"; + } else if (type_it->second == TYPE_SUMMARY) { + ss << "# TYPE nageru_" << type_it->first << " summary\n"; + } + ++type_it; + } + + if (metric.data_type == DATA_TYPE_INT64) { + ss << name << " " << metric.location_int64->load() << "\n"; + } else if (metric.data_type == DATA_TYPE_DOUBLE) { + double val = metric.location_double->load(); + if (isnan(val)) { + // Prometheus can't handle “-nan”. + ss << name << " NaN\n"; + } else { + ss << name << " " << val << "\n"; + } + } else if (metric.data_type == DATA_TYPE_HISTOGRAM) { + ss << metric.location_histogram->serialize(metric.laziness, key_and_metric.first.name, key_and_metric.first.labels); + } else { + ss << metric.location_summary->serialize(metric.laziness, key_and_metric.first.name, key_and_metric.first.labels); + } + } + + return ss.str(); +} + +void Histogram::init(const vector &bucket_vals) +{ + this->num_buckets = bucket_vals.size(); + buckets.reset(new Bucket[num_buckets]); + for (size_t i = 0; i < num_buckets; ++i) { + buckets[i].val = bucket_vals[i]; + } +} + +void Histogram::init_uniform(size_t num_buckets) +{ + this->num_buckets = num_buckets; + buckets.reset(new Bucket[num_buckets]); + for (size_t i = 0; i < num_buckets; ++i) { + buckets[i].val = i; + } +} + +void Histogram::init_geometric(double min, double max, size_t num_buckets) +{ + this->num_buckets = num_buckets; + buckets.reset(new Bucket[num_buckets]); + for (size_t i = 0; i < num_buckets; ++i) { + buckets[i].val = min * pow(max / min, double(i) / (num_buckets - 1)); + } +} + +void Histogram::count_event(double val) +{ + Bucket ref_bucket; + ref_bucket.val = val; + auto it = lower_bound(buckets.get(), buckets.get() + num_buckets, ref_bucket, + [](const Bucket &a, const Bucket &b) { return a.val < b.val; }); + if (it == buckets.get() + num_buckets) { + ++count_after_last_bucket; + } else { + ++it->count; + } + // Non-atomic add, but that's fine, since there are no concurrent writers. + sum = sum + val; +} + +string Histogram::serialize(Metrics::Laziness laziness, const string &name, const vector> &labels) const +{ + // Check if the histogram is empty and should not be serialized. + if (laziness == Metrics::PRINT_WHEN_NONEMPTY && count_after_last_bucket.load() == 0) { + bool empty = true; + for (size_t bucket_idx = 0; bucket_idx < num_buckets; ++bucket_idx) { + if (buckets[bucket_idx].count.load() != 0) { + empty = false; + break; + } + } + if (empty) { + return ""; + } + } + + stringstream ss; + ss.imbue(locale("C")); + ss.precision(20); + + int64_t count = 0; + for (size_t bucket_idx = 0; bucket_idx < num_buckets; ++bucket_idx) { + stringstream le_ss; + le_ss.imbue(locale("C")); + le_ss.precision(20); + le_ss << buckets[bucket_idx].val; + vector> bucket_labels = labels; + bucket_labels.emplace_back("le", le_ss.str()); + + count += buckets[bucket_idx].count.load(); + ss << Metrics::serialize_name(name + "_bucket", bucket_labels) << " " << count << "\n"; + } + + count += count_after_last_bucket.load(); + + ss << Metrics::serialize_name(name + "_sum", labels) << " " << sum.load() << "\n"; + ss << Metrics::serialize_name(name + "_count", labels) << " " << count << "\n"; + + return ss.str(); +} + +void Summary::init(const vector &quantiles, double window_seconds) +{ + this->quantiles = quantiles; + window = duration(window_seconds); +} + +void Summary::count_event(double val) +{ + steady_clock::time_point now = steady_clock::now(); + steady_clock::time_point cutoff = now - duration_cast(window); + + lock_guard lock(mu); + values.emplace_back(now, val); + while (!values.empty() && values.front().first < cutoff) { + values.pop_front(); + } + + // Non-atomic add, but that's fine, since there are no concurrent writers. + sum = sum + val; + ++count; +} + +string Summary::serialize(Metrics::Laziness laziness, const string &name, const vector> &labels) +{ + steady_clock::time_point now = steady_clock::now(); + steady_clock::time_point cutoff = now - duration_cast(window); + + vector values_copy; + { + lock_guard lock(mu); + while (!values.empty() && values.front().first < cutoff) { + values.pop_front(); + } + values_copy.reserve(values.size()); + for (const auto &time_and_value : values) { + values_copy.push_back(time_and_value.second); + } + } + + vector> answers; + if (values_copy.size() == 0) { + if (laziness == Metrics::PRINT_WHEN_NONEMPTY) { + return ""; + } + for (double quantile : quantiles) { + answers.emplace_back(quantile, 0.0 / 0.0); + } + } else if (values_copy.size() == 1) { + for (double quantile : quantiles) { + answers.emplace_back(quantile, values_copy[0]); + } + } else { + // We could probably do repeated nth_element, but the constant factor + // gets a bit high, so just sorting probably is about as fast. + sort(values_copy.begin(), values_copy.end()); + for (double quantile : quantiles) { + double idx = quantile * (values_copy.size() - 1); + size_t idx_floor = size_t(floor(idx)); + const double v0 = values_copy[idx_floor]; + + if (idx_floor == values_copy.size() - 1) { + answers.emplace_back(quantile, values_copy[idx_floor]); + } else { + // Linear interpolation. + double t = idx - idx_floor; + const double v1 = values_copy[idx_floor + 1]; + answers.emplace_back(quantile, v0 + t * (v1 - v0)); + } + } + } + + stringstream ss; + ss.imbue(locale("C")); + ss.precision(20); + + for (const auto &quantile_and_value : answers) { + stringstream quantile_ss; + quantile_ss.imbue(locale("C")); + quantile_ss.precision(3); + quantile_ss << quantile_and_value.first; + vector> quantile_labels = labels; + quantile_labels.emplace_back("quantile", quantile_ss.str()); + + double val = quantile_and_value.second;; + if (isnan(val)) { + // Prometheus can't handle “-nan”. + ss << Metrics::serialize_name(name, quantile_labels) << " NaN\n"; + } else { + ss << Metrics::serialize_name(name, quantile_labels) << " " << val << "\n"; + } + } + + ss << Metrics::serialize_name(name + "_sum", labels) << " " << sum.load() << "\n"; + ss << Metrics::serialize_name(name + "_count", labels) << " " << count.load() << "\n"; + return ss.str(); +} diff --git a/nageru/metrics.h b/nageru/metrics.h new file mode 100644 index 0000000..e2e1e74 --- /dev/null +++ b/nageru/metrics.h @@ -0,0 +1,164 @@ +#ifndef _METRICS_H +#define _METRICS_H 1 + +// A simple global class to keep track of metrics export in Prometheus format. +// It would be better to use a more full-featured Prometheus client library for this, +// but it would introduce a dependency that is not commonly packaged in distributions, +// which makes it quite unwieldy. Thus, we'll package our own for the time being. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class Histogram; +class Summary; + +// Prometheus recommends the use of timestamps instead of “time since event”, +// so you can use this to get the number of seconds since the epoch. +// Note that this will be wrong if your clock changes, so for non-metric use, +// you should use std::chrono::steady_clock instead. +double get_timestamp_for_metrics(); + +class Metrics { +public: + enum Type { + TYPE_COUNTER, + TYPE_GAUGE, + TYPE_HISTOGRAM, // Internal use only. + TYPE_SUMMARY, // Internal use only. + }; + enum Laziness { + PRINT_ALWAYS, + PRINT_WHEN_NONEMPTY, + }; + + void add(const std::string &name, std::atomic *location, Type type = TYPE_COUNTER) + { + add(name, {}, location, type); + } + + void add(const std::string &name, std::atomic *location, Type type = TYPE_COUNTER) + { + add(name, {}, location, type); + } + + void add(const std::string &name, Histogram *location) + { + add(name, {}, location); + } + + void add(const std::string &name, Summary *location) + { + add(name, {}, location); + } + + void add(const std::string &name, const std::vector> &labels, std::atomic *location, Type type = TYPE_COUNTER); + void add(const std::string &name, const std::vector> &labels, std::atomic *location, Type type = TYPE_COUNTER); + void add(const std::string &name, const std::vector> &labels, Histogram *location, Laziness laziness = PRINT_ALWAYS); + void add(const std::string &name, const std::vector> &labels, Summary *location, Laziness laziness = PRINT_ALWAYS); + + void remove(const std::string &name) + { + remove(name, {}); + } + + void remove(const std::string &name, const std::vector> &labels); + + std::string serialize() const; + +private: + static std::string serialize_name(const std::string &name, const std::vector> &labels); + static std::string serialize_labels(const std::vector> &labels); + + enum DataType { + DATA_TYPE_INT64, + DATA_TYPE_DOUBLE, + DATA_TYPE_HISTOGRAM, + DATA_TYPE_SUMMARY, + }; + struct MetricKey { + MetricKey(const std::string &name, const std::vector> labels) + : name(name), labels(labels), serialized_labels(serialize_labels(labels)) + { + } + + bool operator< (const MetricKey &other) const + { + if (name != other.name) + return name < other.name; + return serialized_labels < other.serialized_labels; + } + + const std::string name; + const std::vector> labels; + const std::string serialized_labels; + }; + struct Metric { + DataType data_type; + Laziness laziness; // Only for TYPE_HISTOGRAM. + union { + std::atomic *location_int64; + std::atomic *location_double; + Histogram *location_histogram; + Summary *location_summary; + }; + }; + + mutable std::mutex mu; + std::map types; // Ordered the same as metrics. + std::map metrics; + + friend class Histogram; + friend class Summary; +}; + +class Histogram { +public: + void init(const std::vector &bucket_vals); + void init_uniform(size_t num_buckets); // Sets up buckets 0..(N-1). + void init_geometric(double min, double max, size_t num_buckets); + void count_event(double val); + std::string serialize(Metrics::Laziness laziness, const std::string &name, const std::vector> &labels) const; + +private: + // Bucket counts number of events where val[i - 1] < x <= val[i]. + // The end histogram ends up being made into a cumulative one, + // but that's not how we store it here. + struct Bucket { + double val; + std::atomic count{0}; + }; + std::unique_ptr buckets; + size_t num_buckets; + std::atomic sum{0.0}; + std::atomic count_after_last_bucket{0}; +}; + +// This is a pretty dumb streaming quantile class, but it's exact, and we don't have +// too many values (typically one per frame, and one-minute interval), so we don't +// need anything fancy. +class Summary { +public: + void init(const std::vector &quantiles, double window_seconds); + void count_event(double val); + std::string serialize(Metrics::Laziness laziness, const std::string &name, const std::vector> &labels); + +private: + std::vector quantiles; + std::chrono::duration window; + + mutable std::mutex mu; + std::deque> values; + std::atomic sum{0.0}; + std::atomic count{0}; +}; + +extern Metrics global_metrics; + +#endif // !defined(_METRICS_H) diff --git a/nageru/midi_mapper.cpp b/nageru/midi_mapper.cpp new file mode 100644 index 0000000..3b22192 --- /dev/null +++ b/nageru/midi_mapper.cpp @@ -0,0 +1,676 @@ +#include "midi_mapper.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "audio_mixer.h" +#include "midi_mapping.pb.h" + +using namespace google::protobuf; +using namespace std; +using namespace std::placeholders; + +namespace { + +double map_controller_to_float(int val) +{ + // Slightly hackish mapping so that we can represent exactly 0.0, 0.5 and 1.0. + if (val <= 0) { + return 0.0; + } else if (val >= 127) { + return 1.0; + } else { + return (val + 0.5) / 127.0; + } +} + +} // namespace + +MIDIMapper::MIDIMapper(ControllerReceiver *receiver) + : receiver(receiver), mapping_proto(new MIDIMappingProto) +{ + should_quit_fd = eventfd(/*initval=*/0, /*flags=*/0); + assert(should_quit_fd != -1); +} + +MIDIMapper::~MIDIMapper() +{ + should_quit = true; + const uint64_t one = 1; + if (write(should_quit_fd, &one, sizeof(one)) != sizeof(one)) { + perror("write(should_quit_fd)"); + exit(1); + } + midi_thread.join(); + close(should_quit_fd); +} + +bool load_midi_mapping_from_file(const string &filename, MIDIMappingProto *new_mapping) +{ + // Read and parse the protobuf from disk. + int fd = open(filename.c_str(), O_RDONLY); + if (fd == -1) { + perror(filename.c_str()); + return false; + } + io::FileInputStream input(fd); // Takes ownership of fd. + if (!TextFormat::Parse(&input, new_mapping)) { + input.Close(); + return false; + } + input.Close(); + return true; +} + +bool save_midi_mapping_to_file(const MIDIMappingProto &mapping_proto, const string &filename) +{ + // Save to disk. We use the text format because it's friendlier + // for a user to look at and edit. + int fd = open(filename.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd == -1) { + perror(filename.c_str()); + return false; + } + io::FileOutputStream output(fd); // Takes ownership of fd. + if (!TextFormat::Print(mapping_proto, &output)) { + // TODO: Don't overwrite the old file (if any) on error. + output.Close(); + return false; + } + + output.Close(); + return true; +} + +void MIDIMapper::set_midi_mapping(const MIDIMappingProto &new_mapping) +{ + lock_guard lock(mu); + if (mapping_proto) { + mapping_proto->CopyFrom(new_mapping); + } else { + mapping_proto.reset(new MIDIMappingProto(new_mapping)); + } + + num_controller_banks = min(max(mapping_proto->num_controller_banks(), 1), 5); + current_controller_bank = 0; + + receiver->clear_all_highlights(); + update_highlights(); +} + +void MIDIMapper::start_thread() +{ + midi_thread = thread(&MIDIMapper::thread_func, this); +} + +const MIDIMappingProto &MIDIMapper::get_current_mapping() const +{ + lock_guard lock(mu); + return *mapping_proto; +} + +ControllerReceiver *MIDIMapper::set_receiver(ControllerReceiver *new_receiver) +{ + lock_guard lock(mu); + swap(receiver, new_receiver); + return new_receiver; // Now old receiver. +} + +#define RETURN_ON_ERROR(msg, expr) do { \ + int err = (expr); \ + if (err < 0) { \ + fprintf(stderr, msg ": %s\n", snd_strerror(err)); \ + return; \ + } \ +} while (false) + +#define WARN_ON_ERROR(msg, expr) do { \ + int err = (expr); \ + if (err < 0) { \ + fprintf(stderr, msg ": %s\n", snd_strerror(err)); \ + } \ +} while (false) + + +void MIDIMapper::thread_func() +{ + pthread_setname_np(pthread_self(), "MIDIMapper"); + + snd_seq_t *seq; + int err; + + RETURN_ON_ERROR("snd_seq_open", snd_seq_open(&seq, "default", SND_SEQ_OPEN_DUPLEX, 0)); + RETURN_ON_ERROR("snd_seq_nonblock", snd_seq_nonblock(seq, 1)); + RETURN_ON_ERROR("snd_seq_client_name", snd_seq_set_client_name(seq, "nageru")); + RETURN_ON_ERROR("snd_seq_create_simple_port", + snd_seq_create_simple_port(seq, "nageru", + SND_SEQ_PORT_CAP_READ | + SND_SEQ_PORT_CAP_SUBS_READ | + SND_SEQ_PORT_CAP_WRITE | + SND_SEQ_PORT_CAP_SUBS_WRITE, + SND_SEQ_PORT_TYPE_MIDI_GENERIC | + SND_SEQ_PORT_TYPE_APPLICATION)); + + int queue_id = snd_seq_alloc_queue(seq); + RETURN_ON_ERROR("snd_seq_create_queue", queue_id); + RETURN_ON_ERROR("snd_seq_start_queue", snd_seq_start_queue(seq, queue_id, nullptr)); + + // The sequencer object is now ready to be used from other threads. + { + lock_guard lock(mu); + alsa_seq = seq; + alsa_queue_id = queue_id; + } + + // Listen to the announce port (0:1), which will tell us about new ports. + RETURN_ON_ERROR("snd_seq_connect_from", snd_seq_connect_from(seq, 0, /*client=*/0, /*port=*/1)); + + // Now go through all ports and subscribe to them. + snd_seq_client_info_t *cinfo; + snd_seq_client_info_alloca(&cinfo); + + snd_seq_client_info_set_client(cinfo, -1); + while (snd_seq_query_next_client(seq, cinfo) >= 0) { + int client = snd_seq_client_info_get_client(cinfo); + + snd_seq_port_info_t *pinfo; + snd_seq_port_info_alloca(&pinfo); + + snd_seq_port_info_set_client(pinfo, client); + snd_seq_port_info_set_port(pinfo, -1); + while (snd_seq_query_next_port(seq, pinfo) >= 0) { + constexpr int mask = SND_SEQ_PORT_CAP_READ | SND_SEQ_PORT_CAP_SUBS_READ; + if ((snd_seq_port_info_get_capability(pinfo) & mask) == mask) { + lock_guard lock(mu); + subscribe_to_port_lock_held(seq, *snd_seq_port_info_get_addr(pinfo)); + } + } + } + + int num_alsa_fds = snd_seq_poll_descriptors_count(seq, POLLIN); + unique_ptr fds(new pollfd[num_alsa_fds + 1]); + + while (!should_quit) { + snd_seq_poll_descriptors(seq, fds.get(), num_alsa_fds, POLLIN); + fds[num_alsa_fds].fd = should_quit_fd; + fds[num_alsa_fds].events = POLLIN; + fds[num_alsa_fds].revents = 0; + + err = poll(fds.get(), num_alsa_fds + 1, -1); + if (err == 0 || (err == -1 && errno == EINTR)) { + continue; + } + if (err == -1) { + perror("poll"); + break; + } + if (fds[num_alsa_fds].revents) { + // Activity on should_quit_fd. + break; + } + + // Seemingly we can get multiple events in a single poll, + // and if we don't handle them all, poll will _not_ alert us! + while (!should_quit) { + snd_seq_event_t *event; + err = snd_seq_event_input(seq, &event); + if (err < 0) { + if (err == -EINTR) continue; + if (err == -EAGAIN) break; + if (err == -ENOSPC) { + fprintf(stderr, "snd_seq_event_input: Some events were lost.\n"); + continue; + } + fprintf(stderr, "snd_seq_event_input: %s\n", snd_strerror(err)); + return; + } + if (event) { + handle_event(seq, event); + } + } + } +} + +void MIDIMapper::handle_event(snd_seq_t *seq, snd_seq_event_t *event) +{ + if (event->source.client == snd_seq_client_id(seq)) { + // Ignore events we sent out ourselves. + return; + } + + lock_guard lock(mu); + switch (event->type) { + case SND_SEQ_EVENT_CONTROLLER: { + const int controller = event->data.control.param; + const float value = map_controller_to_float(event->data.control.value); + + receiver->controller_changed(controller); + + // Global controllers. + match_controller(controller, MIDIMappingBusProto::kLocutFieldNumber, MIDIMappingProto::kLocutBankFieldNumber, + value, bind(&ControllerReceiver::set_locut, receiver, _2)); + match_controller(controller, MIDIMappingBusProto::kLimiterThresholdFieldNumber, MIDIMappingProto::kLimiterThresholdBankFieldNumber, + value, bind(&ControllerReceiver::set_limiter_threshold, receiver, _2)); + match_controller(controller, MIDIMappingBusProto::kMakeupGainFieldNumber, MIDIMappingProto::kMakeupGainBankFieldNumber, + value, bind(&ControllerReceiver::set_makeup_gain, receiver, _2)); + + // Bus controllers. + match_controller(controller, MIDIMappingBusProto::kStereoWidthFieldNumber, MIDIMappingProto::kStereoWidthBankFieldNumber, + value, bind(&ControllerReceiver::set_stereo_width, receiver, _1, _2)); + match_controller(controller, MIDIMappingBusProto::kTrebleFieldNumber, MIDIMappingProto::kTrebleBankFieldNumber, + value, bind(&ControllerReceiver::set_treble, receiver, _1, _2)); + match_controller(controller, MIDIMappingBusProto::kMidFieldNumber, MIDIMappingProto::kMidBankFieldNumber, + value, bind(&ControllerReceiver::set_mid, receiver, _1, _2)); + match_controller(controller, MIDIMappingBusProto::kBassFieldNumber, MIDIMappingProto::kBassBankFieldNumber, + value, bind(&ControllerReceiver::set_bass, receiver, _1, _2)); + match_controller(controller, MIDIMappingBusProto::kGainFieldNumber, MIDIMappingProto::kGainBankFieldNumber, + value, bind(&ControllerReceiver::set_gain, receiver, _1, _2)); + match_controller(controller, MIDIMappingBusProto::kCompressorThresholdFieldNumber, MIDIMappingProto::kCompressorThresholdBankFieldNumber, + value, bind(&ControllerReceiver::set_compressor_threshold, receiver, _1, _2)); + match_controller(controller, MIDIMappingBusProto::kFaderFieldNumber, MIDIMappingProto::kFaderBankFieldNumber, + value, bind(&ControllerReceiver::set_fader, receiver, _1, _2)); + break; + } + case SND_SEQ_EVENT_NOTEON: { + const int note = event->data.note.note; + + receiver->note_on(note); + + for (size_t bus_idx = 0; bus_idx < size_t(mapping_proto->bus_mapping_size()); ++bus_idx) { + const MIDIMappingBusProto &bus_mapping = mapping_proto->bus_mapping(bus_idx); + if (bus_mapping.has_prev_bank() && + bus_mapping.prev_bank().note_number() == note) { + current_controller_bank = (current_controller_bank + num_controller_banks - 1) % num_controller_banks; + update_highlights(); + update_lights_lock_held(); + } + if (bus_mapping.has_next_bank() && + bus_mapping.next_bank().note_number() == note) { + current_controller_bank = (current_controller_bank + 1) % num_controller_banks; + update_highlights(); + update_lights_lock_held(); + } + if (bus_mapping.has_select_bank_1() && + bus_mapping.select_bank_1().note_number() == note) { + current_controller_bank = 0; + update_highlights(); + update_lights_lock_held(); + } + if (bus_mapping.has_select_bank_2() && + bus_mapping.select_bank_2().note_number() == note && + num_controller_banks >= 2) { + current_controller_bank = 1; + update_highlights(); + update_lights_lock_held(); + } + if (bus_mapping.has_select_bank_3() && + bus_mapping.select_bank_3().note_number() == note && + num_controller_banks >= 3) { + current_controller_bank = 2; + update_highlights(); + update_lights_lock_held(); + } + if (bus_mapping.has_select_bank_4() && + bus_mapping.select_bank_4().note_number() == note && + num_controller_banks >= 4) { + current_controller_bank = 3; + update_highlights(); + update_lights_lock_held(); + } + if (bus_mapping.has_select_bank_5() && + bus_mapping.select_bank_5().note_number() == note && + num_controller_banks >= 5) { + current_controller_bank = 4; + update_highlights(); + update_lights_lock_held(); + } + } + + match_button(note, MIDIMappingBusProto::kToggleLocutFieldNumber, MIDIMappingProto::kToggleLocutBankFieldNumber, + bind(&ControllerReceiver::toggle_locut, receiver, _1)); + match_button(note, MIDIMappingBusProto::kToggleAutoGainStagingFieldNumber, MIDIMappingProto::kToggleAutoGainStagingBankFieldNumber, + bind(&ControllerReceiver::toggle_auto_gain_staging, receiver, _1)); + match_button(note, MIDIMappingBusProto::kToggleCompressorFieldNumber, MIDIMappingProto::kToggleCompressorBankFieldNumber, + bind(&ControllerReceiver::toggle_compressor, receiver, _1)); + match_button(note, MIDIMappingBusProto::kClearPeakFieldNumber, MIDIMappingProto::kClearPeakBankFieldNumber, + bind(&ControllerReceiver::clear_peak, receiver, _1)); + match_button(note, MIDIMappingBusProto::kToggleMuteFieldNumber, MIDIMappingProto::kClearPeakBankFieldNumber, + bind(&ControllerReceiver::toggle_mute, receiver, _1)); + match_button(note, MIDIMappingBusProto::kToggleLimiterFieldNumber, MIDIMappingProto::kToggleLimiterBankFieldNumber, + bind(&ControllerReceiver::toggle_limiter, receiver)); + match_button(note, MIDIMappingBusProto::kToggleAutoMakeupGainFieldNumber, MIDIMappingProto::kToggleAutoMakeupGainBankFieldNumber, + bind(&ControllerReceiver::toggle_auto_makeup_gain, receiver)); + break; + } + case SND_SEQ_EVENT_PORT_START: + subscribe_to_port_lock_held(seq, event->data.addr); + break; + case SND_SEQ_EVENT_PORT_EXIT: + printf("MIDI port %d:%d went away.\n", event->data.addr.client, event->data.addr.port); + break; + case SND_SEQ_EVENT_PORT_SUBSCRIBED: + if (event->data.connect.sender.client != 0 && // Ignore system senders. + event->data.connect.sender.client != snd_seq_client_id(seq) && + event->data.connect.dest.client == snd_seq_client_id(seq)) { + ++num_subscribed_ports; + update_highlights(); + } + break; + case SND_SEQ_EVENT_PORT_UNSUBSCRIBED: + if (event->data.connect.sender.client != 0 && // Ignore system senders. + event->data.connect.sender.client != snd_seq_client_id(seq) && + event->data.connect.dest.client == snd_seq_client_id(seq)) { + --num_subscribed_ports; + update_highlights(); + } + break; + case SND_SEQ_EVENT_NOTEOFF: + case SND_SEQ_EVENT_CLIENT_START: + case SND_SEQ_EVENT_CLIENT_EXIT: + case SND_SEQ_EVENT_CLIENT_CHANGE: + case SND_SEQ_EVENT_PORT_CHANGE: + break; + default: + printf("Ignoring MIDI event of unknown type %d.\n", event->type); + } +} + +void MIDIMapper::subscribe_to_port_lock_held(snd_seq_t *seq, const snd_seq_addr_t &addr) +{ + // Client 0 (SNDRV_SEQ_CLIENT_SYSTEM) is basically the system; ignore it. + // MIDI through (SNDRV_SEQ_CLIENT_DUMMY) echoes back what we give it, so ignore that, too. + if (addr.client == 0 || addr.client == 14) { + return; + } + + // Don't listen to ourselves. + if (addr.client == snd_seq_client_id(seq)) { + return; + } + + int err = snd_seq_connect_from(seq, 0, addr.client, addr.port); + if (err < 0) { + // Just print out a warning (i.e., don't die); it could + // very well just be e.g. another application. + printf("Couldn't subscribe to MIDI port %d:%d (%s).\n", + addr.client, addr.port, snd_strerror(err)); + } else { + printf("Subscribed to MIDI port %d:%d.\n", addr.client, addr.port); + } + + // For sending data back. + err = snd_seq_connect_to(seq, 0, addr.client, addr.port); + if (err < 0) { + printf("Couldn't subscribe MIDI port %d:%d (%s) to us.\n", + addr.client, addr.port, snd_strerror(err)); + } else { + printf("Subscribed MIDI port %d:%d to us.\n", addr.client, addr.port); + } + + current_light_status.clear(); // The current state of the device is unknown. + update_lights_lock_held(); +} + +void MIDIMapper::match_controller(int controller, int field_number, int bank_field_number, float value, function func) +{ + if (bank_mismatch(bank_field_number)) { + return; + } + + for (size_t bus_idx = 0; bus_idx < size_t(mapping_proto->bus_mapping_size()); ++bus_idx) { + const MIDIMappingBusProto &bus_mapping = mapping_proto->bus_mapping(bus_idx); + + const FieldDescriptor *descriptor = bus_mapping.GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping.GetReflection(); + if (!bus_reflection->HasField(bus_mapping, descriptor)) { + continue; + } + const MIDIControllerProto &controller_proto = + static_cast(bus_reflection->GetMessage(bus_mapping, descriptor)); + if (controller_proto.controller_number() == controller) { + func(bus_idx, value); + } + } +} + +void MIDIMapper::match_button(int note, int field_number, int bank_field_number, function func) +{ + if (bank_mismatch(bank_field_number)) { + return; + } + + for (size_t bus_idx = 0; bus_idx < size_t(mapping_proto->bus_mapping_size()); ++bus_idx) { + const MIDIMappingBusProto &bus_mapping = mapping_proto->bus_mapping(bus_idx); + + const FieldDescriptor *descriptor = bus_mapping.GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping.GetReflection(); + if (!bus_reflection->HasField(bus_mapping, descriptor)) { + continue; + } + const MIDIButtonProto &button_proto = + static_cast(bus_reflection->GetMessage(bus_mapping, descriptor)); + if (button_proto.note_number() == note) { + func(bus_idx); + } + } +} + +bool MIDIMapper::has_active_controller(unsigned bus_idx, int field_number, int bank_field_number) +{ + if (bank_mismatch(bank_field_number)) { + return false; + } + + const MIDIMappingBusProto &bus_mapping = mapping_proto->bus_mapping(bus_idx); + const FieldDescriptor *descriptor = bus_mapping.GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping.GetReflection(); + return bus_reflection->HasField(bus_mapping, descriptor); +} + +bool MIDIMapper::bank_mismatch(int bank_field_number) +{ + const FieldDescriptor *bank_descriptor = mapping_proto->GetDescriptor()->FindFieldByNumber(bank_field_number); + const Reflection *reflection = mapping_proto->GetReflection(); + return (reflection->HasField(*mapping_proto, bank_descriptor) && + reflection->GetInt32(*mapping_proto, bank_descriptor) != current_controller_bank); +} + +void MIDIMapper::refresh_highlights() +{ + receiver->clear_all_highlights(); + update_highlights(); +} + +void MIDIMapper::refresh_lights() +{ + lock_guard lock(mu); + update_lights_lock_held(); +} + +void MIDIMapper::update_highlights() +{ + if (num_subscribed_ports.load() == 0) { + receiver->clear_all_highlights(); + return; + } + + // Global controllers. + bool highlight_locut = false; + bool highlight_limiter_threshold = false; + bool highlight_makeup_gain = false; + bool highlight_toggle_limiter = false; + bool highlight_toggle_auto_makeup_gain = false; + for (size_t bus_idx = 0; bus_idx < size_t(mapping_proto->bus_mapping_size()); ++bus_idx) { + if (has_active_controller( + bus_idx, MIDIMappingBusProto::kLocutFieldNumber, MIDIMappingProto::kLocutBankFieldNumber)) { + highlight_locut = true; + } + if (has_active_controller( + bus_idx, MIDIMappingBusProto::kLimiterThresholdFieldNumber, MIDIMappingProto::kLimiterThresholdBankFieldNumber)) { + highlight_limiter_threshold = true; + } + if (has_active_controller( + bus_idx, MIDIMappingBusProto::kMakeupGainFieldNumber, MIDIMappingProto::kMakeupGainBankFieldNumber)) { + highlight_makeup_gain = true; + } + if (has_active_controller( + bus_idx, MIDIMappingBusProto::kToggleLimiterFieldNumber, MIDIMappingProto::kToggleLimiterBankFieldNumber)) { + highlight_toggle_limiter = true; + } + if (has_active_controller( + bus_idx, MIDIMappingBusProto::kToggleAutoMakeupGainFieldNumber, MIDIMappingProto::kToggleAutoMakeupGainBankFieldNumber)) { + highlight_toggle_auto_makeup_gain = true; + } + } + receiver->highlight_locut(highlight_locut); + receiver->highlight_limiter_threshold(highlight_limiter_threshold); + receiver->highlight_makeup_gain(highlight_makeup_gain); + receiver->highlight_toggle_limiter(highlight_toggle_limiter); + receiver->highlight_toggle_auto_makeup_gain(highlight_toggle_auto_makeup_gain); + + // Per-bus controllers. + for (size_t bus_idx = 0; bus_idx < size_t(mapping_proto->bus_mapping_size()); ++bus_idx) { + receiver->highlight_stereo_width(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kStereoWidthFieldNumber, MIDIMappingProto::kStereoWidthBankFieldNumber)); + receiver->highlight_treble(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kTrebleFieldNumber, MIDIMappingProto::kTrebleBankFieldNumber)); + receiver->highlight_mid(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kMidFieldNumber, MIDIMappingProto::kMidBankFieldNumber)); + receiver->highlight_bass(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kBassFieldNumber, MIDIMappingProto::kBassBankFieldNumber)); + receiver->highlight_gain(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kGainFieldNumber, MIDIMappingProto::kGainBankFieldNumber)); + receiver->highlight_compressor_threshold(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kCompressorThresholdFieldNumber, MIDIMappingProto::kCompressorThresholdBankFieldNumber)); + receiver->highlight_fader(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kFaderFieldNumber, MIDIMappingProto::kFaderBankFieldNumber)); + receiver->highlight_mute(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kToggleMuteFieldNumber, MIDIMappingProto::kToggleMuteBankFieldNumber)); + receiver->highlight_toggle_locut(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kToggleLocutFieldNumber, MIDIMappingProto::kToggleLocutBankFieldNumber)); + receiver->highlight_toggle_auto_gain_staging(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kToggleAutoGainStagingFieldNumber, MIDIMappingProto::kToggleAutoGainStagingBankFieldNumber)); + receiver->highlight_toggle_compressor(bus_idx, has_active_controller( + bus_idx, MIDIMappingBusProto::kToggleCompressorFieldNumber, MIDIMappingProto::kToggleCompressorBankFieldNumber)); + } +} + +void MIDIMapper::update_lights_lock_held() +{ + if (alsa_seq == nullptr || global_audio_mixer == nullptr) { + return; + } + + set active_lights; // Desired state. + if (current_controller_bank == 0) { + activate_lights_all_buses(MIDIMappingBusProto::kBank1IsSelectedFieldNumber, &active_lights); + } + if (current_controller_bank == 1) { + activate_lights_all_buses(MIDIMappingBusProto::kBank2IsSelectedFieldNumber, &active_lights); + } + if (current_controller_bank == 2) { + activate_lights_all_buses(MIDIMappingBusProto::kBank3IsSelectedFieldNumber, &active_lights); + } + if (current_controller_bank == 3) { + activate_lights_all_buses(MIDIMappingBusProto::kBank4IsSelectedFieldNumber, &active_lights); + } + if (current_controller_bank == 4) { + activate_lights_all_buses(MIDIMappingBusProto::kBank5IsSelectedFieldNumber, &active_lights); + } + if (global_audio_mixer->get_limiter_enabled()) { + activate_lights_all_buses(MIDIMappingBusProto::kLimiterIsOnFieldNumber, &active_lights); + } + if (global_audio_mixer->get_final_makeup_gain_auto()) { + activate_lights_all_buses(MIDIMappingBusProto::kAutoMakeupGainIsOnFieldNumber, &active_lights); + } + unsigned num_buses = min(global_audio_mixer->num_buses(), mapping_proto->bus_mapping_size()); + for (unsigned bus_idx = 0; bus_idx < num_buses; ++bus_idx) { + if (global_audio_mixer->get_mute(bus_idx)) { + activate_lights(bus_idx, MIDIMappingBusProto::kIsMutedFieldNumber, &active_lights); + } + if (global_audio_mixer->get_locut_enabled(bus_idx)) { + activate_lights(bus_idx, MIDIMappingBusProto::kLocutIsOnFieldNumber, &active_lights); + } + if (global_audio_mixer->get_gain_staging_auto(bus_idx)) { + activate_lights(bus_idx, MIDIMappingBusProto::kAutoGainStagingIsOnFieldNumber, &active_lights); + } + if (global_audio_mixer->get_compressor_enabled(bus_idx)) { + activate_lights(bus_idx, MIDIMappingBusProto::kCompressorIsOnFieldNumber, &active_lights); + } + if (has_peaked[bus_idx]) { + activate_lights(bus_idx, MIDIMappingBusProto::kHasPeakedFieldNumber, &active_lights); + } + } + + unsigned num_events = 0; + for (unsigned note_num = 1; note_num <= 127; ++note_num) { + bool active = active_lights.count(note_num); + if (current_light_status.count(note_num) && + current_light_status[note_num] == active) { + // Already known to be in the desired state. + continue; + } + + snd_seq_event_t ev; + snd_seq_ev_clear(&ev); + + // Some devices drop events if we throw them onto them + // too quickly. Add a 1 ms delay for each. + snd_seq_real_time_t tm{0, num_events++ * 1000000}; + snd_seq_ev_schedule_real(&ev, alsa_queue_id, true, &tm); + snd_seq_ev_set_source(&ev, 0); + snd_seq_ev_set_subs(&ev); + + // For some reason, not all devices respond to note off. + // Use note-on with velocity of 0 (which is equivalent) instead. + snd_seq_ev_set_noteon(&ev, /*channel=*/0, note_num, active ? 127 : 0); + WARN_ON_ERROR("snd_seq_event_output", snd_seq_event_output(alsa_seq, &ev)); + current_light_status[note_num] = active; + } + WARN_ON_ERROR("snd_seq_drain_output", snd_seq_drain_output(alsa_seq)); +} + +void MIDIMapper::activate_lights(unsigned bus_idx, int field_number, set *active_lights) +{ + const MIDIMappingBusProto &bus_mapping = mapping_proto->bus_mapping(bus_idx); + + const FieldDescriptor *descriptor = bus_mapping.GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping.GetReflection(); + if (!bus_reflection->HasField(bus_mapping, descriptor)) { + return; + } + const MIDILightProto &light_proto = + static_cast(bus_reflection->GetMessage(bus_mapping, descriptor)); + active_lights->insert(light_proto.note_number()); +} + +void MIDIMapper::activate_lights_all_buses(int field_number, set *active_lights) +{ + for (size_t bus_idx = 0; bus_idx < size_t(mapping_proto->bus_mapping_size()); ++bus_idx) { + const MIDIMappingBusProto &bus_mapping = mapping_proto->bus_mapping(bus_idx); + + const FieldDescriptor *descriptor = bus_mapping.GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping.GetReflection(); + if (!bus_reflection->HasField(bus_mapping, descriptor)) { + continue; + } + const MIDILightProto &light_proto = + static_cast(bus_reflection->GetMessage(bus_mapping, descriptor)); + active_lights->insert(light_proto.note_number()); + } +} diff --git a/nageru/midi_mapper.h b/nageru/midi_mapper.h new file mode 100644 index 0000000..42bf19a --- /dev/null +++ b/nageru/midi_mapper.h @@ -0,0 +1,137 @@ +#ifndef _MIDI_MAPPER_H +#define _MIDI_MAPPER_H 1 + +// MIDIMapper is a class that listens for incoming MIDI messages from +// mixer controllers (ie., it is not meant to be used with regular +// instruments), interprets them according to a device-specific, user-defined +// mapping, and calls back into a receiver (typically the MainWindow). +// This way, it is possible to control audio functionality using physical +// pots and faders instead of the mouse. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "defs.h" + +class MIDIMappingProto; +typedef struct snd_seq_addr snd_seq_addr_t; +typedef struct snd_seq_event snd_seq_event_t; +typedef struct _snd_seq snd_seq_t; + +// Interface for receiving interpreted controller messages. +class ControllerReceiver { +public: + virtual ~ControllerReceiver() {} + + // All values are [0.0, 1.0]. + virtual void set_locut(float value) = 0; + virtual void set_limiter_threshold(float value) = 0; + virtual void set_makeup_gain(float value) = 0; + + virtual void set_stereo_width(unsigned bus_idx, float value) = 0; + virtual void set_treble(unsigned bus_idx, float value) = 0; + virtual void set_mid(unsigned bus_idx, float value) = 0; + virtual void set_bass(unsigned bus_idx, float value) = 0; + virtual void set_gain(unsigned bus_idx, float value) = 0; + virtual void set_compressor_threshold(unsigned bus_idx, float value) = 0; + virtual void set_fader(unsigned bus_idx, float value) = 0; + + virtual void toggle_mute(unsigned bus_idx) = 0; + virtual void toggle_locut(unsigned bus_idx) = 0; + virtual void toggle_auto_gain_staging(unsigned bus_idx) = 0; + virtual void toggle_compressor(unsigned bus_idx) = 0; + virtual void clear_peak(unsigned bus_idx) = 0; + virtual void toggle_limiter() = 0; + virtual void toggle_auto_makeup_gain() = 0; + + // Signals to highlight controls to mark them to the user + // as MIDI-controllable (or not). + virtual void clear_all_highlights() = 0; + + virtual void highlight_locut(bool highlight) = 0; + virtual void highlight_limiter_threshold(bool highlight) = 0; + virtual void highlight_makeup_gain(bool highlight) = 0; + + virtual void highlight_stereo_width(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_treble(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_mid(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_bass(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_gain(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_compressor_threshold(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_fader(unsigned bus_idx, bool highlight) = 0; + + virtual void highlight_mute(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_toggle_locut(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_toggle_auto_gain_staging(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_toggle_compressor(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_clear_peak(unsigned bus_idx, bool highlight) = 0; + virtual void highlight_toggle_limiter(bool highlight) = 0; + virtual void highlight_toggle_auto_makeup_gain(bool highlight) = 0; + + // Raw events; used for the editor dialog only. + virtual void controller_changed(unsigned controller) = 0; + virtual void note_on(unsigned note) = 0; +}; + +class MIDIMapper { +public: + MIDIMapper(ControllerReceiver *receiver); + virtual ~MIDIMapper(); + void set_midi_mapping(const MIDIMappingProto &new_mapping); + void start_thread(); + const MIDIMappingProto &get_current_mapping() const; + + // Overwrites and returns the previous value. + ControllerReceiver *set_receiver(ControllerReceiver *new_receiver); + + void refresh_highlights(); + void refresh_lights(); + + void set_has_peaked(unsigned bus_idx, bool has_peaked) + { + this->has_peaked[bus_idx] = has_peaked; + } + +private: + void thread_func(); + void handle_event(snd_seq_t *seq, snd_seq_event_t *event); + void subscribe_to_port_lock_held(snd_seq_t *seq, const snd_seq_addr_t &addr); + void match_controller(int controller, int field_number, int bank_field_number, float value, std::function func); + void match_button(int note, int field_number, int bank_field_number, std::function func); + bool has_active_controller(unsigned bus_idx, int field_number, int bank_field_number); // Also works for buttons. + bool bank_mismatch(int bank_field_number); + + void update_highlights(); + + void update_lights_lock_held(); + void activate_lights(unsigned bus_idx, int field_number, std::set *active_lights); + void activate_lights_all_buses(int field_number, std::set *active_lights); + + std::atomic should_quit{false}; + int should_quit_fd; + + std::atomic has_peaked[MAX_BUSES] {{ false }}; + + mutable std::mutex mu; + ControllerReceiver *receiver; // Under . + std::unique_ptr mapping_proto; // Under . + int num_controller_banks; // Under . + std::atomic current_controller_bank{0}; + std::atomic num_subscribed_ports{0}; + + std::thread midi_thread; + std::map current_light_status; // Keyed by note number. Under . + snd_seq_t *alsa_seq{nullptr}; // Under . + int alsa_queue_id{-1}; // Under . +}; + +bool load_midi_mapping_from_file(const std::string &filename, MIDIMappingProto *new_mapping); +bool save_midi_mapping_to_file(const MIDIMappingProto &mapping_proto, const std::string &filename); + +#endif // !defined(_MIDI_MAPPER_H) diff --git a/nageru/midi_mapping.proto b/nageru/midi_mapping.proto new file mode 100644 index 0000000..4a8b852 --- /dev/null +++ b/nageru/midi_mapping.proto @@ -0,0 +1,119 @@ +// Mappings from MIDI controllers to the UI. (We don't really build +// a more complicated data structure than this in Nageru itself either; +// we just edit and match directly against the protobuf.) + +syntax = "proto2"; + +// A single, given controller mapping. +message MIDIControllerProto { + required int32 controller_number = 1; + // TODO: Add flags like invert here if/when we need them. +} + +message MIDIButtonProto { + required int32 note_number = 1; +} + +message MIDILightProto { + required int32 note_number = 1; +} + +// All the mappings for a given a bus. +message MIDIMappingBusProto { + // TODO: If we need support for lots of buses (i.e., more than the typical eight + // on a mixer), add a system for bus banks, like we have for controller banks. + // optional int32 bus_bank = 1; + + optional MIDIControllerProto stereo_width = 37; + optional MIDIControllerProto treble = 2; + optional MIDIControllerProto mid = 3; + optional MIDIControllerProto bass = 4; + optional MIDIControllerProto gain = 5; + optional MIDIControllerProto compressor_threshold = 6; + optional MIDIControllerProto fader = 7; + + optional MIDIButtonProto toggle_mute = 8; + optional MIDIButtonProto toggle_locut = 9; + optional MIDIButtonProto toggle_auto_gain_staging = 10; + optional MIDIButtonProto toggle_compressor = 11; + optional MIDIButtonProto clear_peak = 12; + + // These are really global (controller bank change affects all buss), + // but it's not uncommon that we'd want one button per bus to switch banks. + // E.g., if the user binds the “mute” button to “next bank”, they'd want every + // mute button on the mixer to do that, so they need one mapping per bus. + optional MIDIButtonProto prev_bank = 13; + optional MIDIButtonProto next_bank = 14; + optional MIDIButtonProto select_bank_1 = 15; + optional MIDIButtonProto select_bank_2 = 16; + optional MIDIButtonProto select_bank_3 = 17; + optional MIDIButtonProto select_bank_4 = 18; + optional MIDIButtonProto select_bank_5 = 19; + optional MIDIButtonProto toggle_limiter = 20; + optional MIDIButtonProto toggle_auto_makeup_gain = 21; + + // These are also global (they belong to the master bus), and unlike + // the bank change commands, one would usually have only one of each, + // but there's no reason to limit them to one each, and the editor UI + // becomes simpler if they are the treated the same way as the bank + // commands. + optional MIDIControllerProto locut = 22; + optional MIDIControllerProto limiter_threshold = 23; + optional MIDIControllerProto makeup_gain = 24; + + // Per-bus lights. + optional MIDILightProto is_muted = 25; + optional MIDILightProto locut_is_on = 26; + optional MIDILightProto auto_gain_staging_is_on = 27; + optional MIDILightProto compressor_is_on = 28; + optional MIDILightProto has_peaked = 29; + + // Global lights. Same logic as above for why they're in this proto. + optional MIDILightProto bank_1_is_selected = 30; + optional MIDILightProto bank_2_is_selected = 31; + optional MIDILightProto bank_3_is_selected = 32; + optional MIDILightProto bank_4_is_selected = 33; + optional MIDILightProto bank_5_is_selected = 34; + optional MIDILightProto limiter_is_on = 35; + optional MIDILightProto auto_makeup_gain_is_on = 36; +} + +// The top-level protobuf, containing all the bus mappings, as well as +// more global settings. +// +// Since a typical mixer will have fewer physical controls than what Nageru +// could use, Nageru supports so-called controller banks. A mapping can +// optionally belong to a bank, and if so, that mapping is only active when +// that bank is selected. The user can then select the current bank using +// other mappings, typically by having some mixer button assigned to +// “next bank”. This yields effective multiplexing of lesser-used controls. +message MIDIMappingProto { + optional int32 num_controller_banks = 1 [default = 0]; // Max 5. + + // Bus controller banks. + optional int32 stereo_width_bank = 19; + optional int32 treble_bank = 2; + optional int32 mid_bank = 3; + optional int32 bass_bank = 4; + optional int32 gain_bank = 5; + optional int32 compressor_threshold_bank = 6; + optional int32 fader_bank = 7; + + // Bus button banks. + optional int32 toggle_mute_bank = 8; + optional int32 toggle_locut_bank = 9; + optional int32 toggle_auto_gain_staging_bank = 10; + optional int32 toggle_compressor_bank = 11; + optional int32 clear_peak_bank = 12; + + // Global controller banks. + optional int32 locut_bank = 13; + optional int32 limiter_threshold_bank = 14; + optional int32 makeup_gain_bank = 15; + + // Global buttons. + optional int32 toggle_limiter_bank = 16; + optional int32 toggle_auto_makeup_gain_bank = 17; + + repeated MIDIMappingBusProto bus_mapping = 18; +} diff --git a/nageru/midi_mapping.ui b/nageru/midi_mapping.ui new file mode 100644 index 0000000..3839858 --- /dev/null +++ b/nageru/midi_mapping.ui @@ -0,0 +1,105 @@ + + + MIDIMappingDialog + + + + 0 + 0 + 879 + 583 + + + + MIDI controller setup + + + + + + + 1 + + + + + + + + Add or change a mapping by clicking in the cell, then moving the corresponding control on your MIDI device. + + + + + + + + + Guess &bus + + + + + + + Guess &group + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + &Save… + + + + + + + &Load… + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + + + + diff --git a/nageru/midi_mapping_dialog.cpp b/nageru/midi_mapping_dialog.cpp new file mode 100644 index 0000000..05508e4 --- /dev/null +++ b/nageru/midi_mapping_dialog.cpp @@ -0,0 +1,612 @@ +#include "midi_mapping_dialog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "midi_mapper.h" +#include "midi_mapping.pb.h" +#include "post_to_main_thread.h" +#include "ui_midi_mapping.h" + +class QObject; + +using namespace google::protobuf; +using namespace std; + +vector per_bus_controllers = { + { "Stereo width", MIDIMappingBusProto::kStereoWidthFieldNumber, + MIDIMappingProto::kStereoWidthBankFieldNumber }, + { "Treble", MIDIMappingBusProto::kTrebleFieldNumber, MIDIMappingProto::kTrebleBankFieldNumber }, + { "Mid", MIDIMappingBusProto::kMidFieldNumber, MIDIMappingProto::kMidBankFieldNumber }, + { "Bass", MIDIMappingBusProto::kBassFieldNumber, MIDIMappingProto::kBassBankFieldNumber }, + { "Gain", MIDIMappingBusProto::kGainFieldNumber, MIDIMappingProto::kGainBankFieldNumber }, + { "Compressor threshold", MIDIMappingBusProto::kCompressorThresholdFieldNumber, + MIDIMappingProto::kCompressorThresholdBankFieldNumber}, + { "Fader", MIDIMappingBusProto::kFaderFieldNumber, MIDIMappingProto::kFaderBankFieldNumber } +}; +vector per_bus_buttons = { + { "Toggle mute", MIDIMappingBusProto::kToggleMuteFieldNumber, + MIDIMappingProto::kToggleMuteBankFieldNumber }, + { "Toggle locut", MIDIMappingBusProto::kToggleLocutFieldNumber, + MIDIMappingProto::kToggleLocutBankFieldNumber }, + { "Togle auto gain staging", MIDIMappingBusProto::kToggleAutoGainStagingFieldNumber, + MIDIMappingProto::kToggleAutoGainStagingBankFieldNumber }, + { "Togle compressor", MIDIMappingBusProto::kToggleCompressorFieldNumber, + MIDIMappingProto::kToggleCompressorBankFieldNumber }, + { "Clear peak", MIDIMappingBusProto::kClearPeakFieldNumber, + MIDIMappingProto::kClearPeakBankFieldNumber } +}; +vector per_bus_lights = { + { "Is muted", MIDIMappingBusProto::kIsMutedFieldNumber, 0 }, + { "Locut is on", MIDIMappingBusProto::kLocutIsOnFieldNumber, 0 }, + { "Auto gain staging is on", MIDIMappingBusProto::kAutoGainStagingIsOnFieldNumber, 0 }, + { "Compressor is on", MIDIMappingBusProto::kCompressorIsOnFieldNumber, 0 }, + { "Bus has peaked", MIDIMappingBusProto::kHasPeakedFieldNumber, 0 } +}; +vector global_controllers = { + { "Locut cutoff", MIDIMappingBusProto::kLocutFieldNumber, MIDIMappingProto::kLocutBankFieldNumber }, + { "Limiter threshold", MIDIMappingBusProto::kLimiterThresholdFieldNumber, + MIDIMappingProto::kLimiterThresholdBankFieldNumber }, + { "Makeup gain", MIDIMappingBusProto::kMakeupGainFieldNumber, + MIDIMappingProto::kMakeupGainBankFieldNumber } +}; +vector global_buttons = { + { "Previous bank", MIDIMappingBusProto::kPrevBankFieldNumber, 0 }, + { "Next bank", MIDIMappingBusProto::kNextBankFieldNumber, 0 }, + { "Select bank 1", MIDIMappingBusProto::kSelectBank1FieldNumber, 0 }, + { "Select bank 2", MIDIMappingBusProto::kSelectBank2FieldNumber, 0 }, + { "Select bank 3", MIDIMappingBusProto::kSelectBank3FieldNumber, 0 }, + { "Select bank 4", MIDIMappingBusProto::kSelectBank4FieldNumber, 0 }, + { "Select bank 5", MIDIMappingBusProto::kSelectBank5FieldNumber, 0 }, + { "Toggle limiter", MIDIMappingBusProto::kToggleLimiterFieldNumber, MIDIMappingProto::kToggleLimiterBankFieldNumber }, + { "Toggle auto makeup gain", MIDIMappingBusProto::kToggleAutoMakeupGainFieldNumber, MIDIMappingProto::kToggleAutoMakeupGainBankFieldNumber } +}; +vector global_lights = { + { "Bank 1 is selected", MIDIMappingBusProto::kBank1IsSelectedFieldNumber, 0 }, + { "Bank 2 is selected", MIDIMappingBusProto::kBank2IsSelectedFieldNumber, 0 }, + { "Bank 3 is selected", MIDIMappingBusProto::kBank3IsSelectedFieldNumber, 0 }, + { "Bank 4 is selected", MIDIMappingBusProto::kBank4IsSelectedFieldNumber, 0 }, + { "Bank 5 is selected", MIDIMappingBusProto::kBank5IsSelectedFieldNumber, 0 }, + { "Limiter is on", MIDIMappingBusProto::kLimiterIsOnFieldNumber, 0 }, + { "Auto makeup gain is on", MIDIMappingBusProto::kAutoMakeupGainIsOnFieldNumber, 0 }, +}; + +namespace { + +int get_bank(const MIDIMappingProto &mapping_proto, int bank_field_number, int default_value) +{ + const FieldDescriptor *bank_descriptor = mapping_proto.GetDescriptor()->FindFieldByNumber(bank_field_number); + const Reflection *reflection = mapping_proto.GetReflection(); + if (!reflection->HasField(mapping_proto, bank_descriptor)) { + return default_value; + } + return reflection->GetInt32(mapping_proto, bank_descriptor); +} + +int get_controller_mapping(const MIDIMappingProto &mapping_proto, size_t bus_idx, int field_number, int default_value) +{ + if (bus_idx >= size_t(mapping_proto.bus_mapping_size())) { + return default_value; + } + + const MIDIMappingBusProto &bus_mapping = mapping_proto.bus_mapping(bus_idx); + const FieldDescriptor *descriptor = bus_mapping.GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping.GetReflection(); + if (!bus_reflection->HasField(bus_mapping, descriptor)) { + return default_value; + } + const MIDIControllerProto &controller_proto = + static_cast(bus_reflection->GetMessage(bus_mapping, descriptor)); + return controller_proto.controller_number(); +} + +int get_button_mapping(const MIDIMappingProto &mapping_proto, size_t bus_idx, int field_number, int default_value) +{ + if (bus_idx >= size_t(mapping_proto.bus_mapping_size())) { + return default_value; + } + + const MIDIMappingBusProto &bus_mapping = mapping_proto.bus_mapping(bus_idx); + const FieldDescriptor *descriptor = bus_mapping.GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping.GetReflection(); + if (!bus_reflection->HasField(bus_mapping, descriptor)) { + return default_value; + } + const MIDIButtonProto &bus_proto = + static_cast(bus_reflection->GetMessage(bus_mapping, descriptor)); + return bus_proto.note_number(); +} + +int get_light_mapping(const MIDIMappingProto &mapping_proto, size_t bus_idx, int field_number, int default_value) +{ + if (bus_idx >= size_t(mapping_proto.bus_mapping_size())) { + return default_value; + } + + const MIDIMappingBusProto &bus_mapping = mapping_proto.bus_mapping(bus_idx); + const FieldDescriptor *descriptor = bus_mapping.GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping.GetReflection(); + if (!bus_reflection->HasField(bus_mapping, descriptor)) { + return default_value; + } + const MIDILightProto &bus_proto = + static_cast(bus_reflection->GetMessage(bus_mapping, descriptor)); + return bus_proto.note_number(); +} + +} // namespace + +MIDIMappingDialog::MIDIMappingDialog(MIDIMapper *mapper) + : ui(new Ui::MIDIMappingDialog), + mapper(mapper) +{ + ui->setupUi(this); + + const MIDIMappingProto mapping_proto = mapper->get_current_mapping(); // Take a copy. + old_receiver = mapper->set_receiver(this); + + QStringList labels; + labels << ""; + labels << "Controller bank"; + for (unsigned bus_idx = 0; bus_idx < num_buses; ++bus_idx) { + char buf[256]; + snprintf(buf, sizeof(buf), "Bus %d", bus_idx + 1); + labels << buf; + } + labels << ""; + ui->treeWidget->setColumnCount(num_buses + 3); + ui->treeWidget->setHeaderLabels(labels); + + add_controls("Per-bus controllers", ControlType::CONTROLLER, SpinnerGroup::PER_BUS_CONTROLLERS, mapping_proto, per_bus_controllers); + add_controls("Per-bus buttons", ControlType::BUTTON, SpinnerGroup::PER_BUS_BUTTONS, mapping_proto, per_bus_buttons); + add_controls("Per-bus lights", ControlType::LIGHT, SpinnerGroup::PER_BUS_LIGHTS, mapping_proto, per_bus_lights); + add_controls("Global controllers", ControlType::CONTROLLER, SpinnerGroup::GLOBAL_CONTROLLERS, mapping_proto, global_controllers); + add_controls("Global buttons", ControlType::BUTTON, SpinnerGroup::GLOBAL_BUTTONS, mapping_proto, global_buttons); + add_controls("Global lights", ControlType::LIGHT, SpinnerGroup::GLOBAL_LIGHTS, mapping_proto, global_lights); + fill_controls_from_mapping(mapping_proto); + + // Auto-resize every column but the last. + for (unsigned column_idx = 0; column_idx < num_buses + 3; ++column_idx) { + ui->treeWidget->resizeColumnToContents(column_idx); + } + + connect(ui->guess_bus_button, &QPushButton::clicked, + bind(&MIDIMappingDialog::guess_clicked, this, false)); + connect(ui->guess_group_button, &QPushButton::clicked, + bind(&MIDIMappingDialog::guess_clicked, this, true)); + connect(ui->ok_cancel_buttons, &QDialogButtonBox::accepted, this, &MIDIMappingDialog::ok_clicked); + connect(ui->ok_cancel_buttons, &QDialogButtonBox::rejected, this, &MIDIMappingDialog::cancel_clicked); + connect(ui->save_button, &QPushButton::clicked, this, &MIDIMappingDialog::save_clicked); + connect(ui->load_button, &QPushButton::clicked, this, &MIDIMappingDialog::load_clicked); + + update_guess_button_state(); +} + +MIDIMappingDialog::~MIDIMappingDialog() +{ + mapper->set_receiver(old_receiver); + mapper->refresh_highlights(); +} + +bool MIDIMappingDialog::eventFilter(QObject *obj, QEvent *event) +{ + if (event->type() == QEvent::FocusIn || + event->type() == QEvent::FocusOut) { + // We ignore the guess buttons themselves; it should be allowed + // to navigate from a spinner to focus on a button (to click it). + if (obj != ui->guess_bus_button && obj != ui->guess_group_button) { + update_guess_button_state(); + } + } + return false; +} + +void MIDIMappingDialog::guess_clicked(bool limit_to_group) +{ + FocusInfo focus = find_focus(); + if (focus.bus_idx == -1) { + // The guess button probably took the focus away from us. + focus = last_focus; + } + assert(focus.bus_idx != -1); // The button should have been disabled. + pair bus_and_offset = guess_offset(focus.bus_idx, limit_to_group ? focus.spinner_group : SpinnerGroup::ALL_GROUPS); + const int source_bus_idx = bus_and_offset.first; + const int offset = bus_and_offset.second; + assert(source_bus_idx != -1); // The button should have been disabled. + + for (const auto &field_number_and_spinner : spinners[focus.bus_idx]) { + int field_number = field_number_and_spinner.first; + QSpinBox *spinner = field_number_and_spinner.second.spinner; + SpinnerGroup this_spinner_group = field_number_and_spinner.second.group; + + if (limit_to_group && this_spinner_group != focus.spinner_group) { + continue; + } + + assert(spinners[source_bus_idx].count(field_number)); + QSpinBox *source_spinner = spinners[source_bus_idx][field_number].spinner; + assert(spinners[source_bus_idx][field_number].group == this_spinner_group); + + if (source_spinner->value() != -1) { + spinner->setValue(source_spinner->value() + offset); + } + } + + // See if we can find a “next” bus to move the focus to. + const int next_bus_idx = focus.bus_idx + (focus.bus_idx - source_bus_idx); // Note: Could become e.g. -1. + for (const InstantiatedSpinner &is : controller_spinners) { + if (int(is.bus_idx) == next_bus_idx && is.field_number == focus.field_number) { + is.spinner->setFocus(); + } + } + for (const InstantiatedSpinner &is : button_spinners) { + if (int(is.bus_idx) == next_bus_idx && is.field_number == focus.field_number) { + is.spinner->setFocus(); + } + } + for (const InstantiatedSpinner &is : light_spinners) { + if (int(is.bus_idx) == next_bus_idx && is.field_number == focus.field_number) { + is.spinner->setFocus(); + } + } +} + +void MIDIMappingDialog::ok_clicked() +{ + unique_ptr new_mapping = construct_mapping_proto_from_ui(); + mapper->set_midi_mapping(*new_mapping); + mapper->set_receiver(old_receiver); + accept(); +} + +void MIDIMappingDialog::cancel_clicked() +{ + mapper->set_receiver(old_receiver); + reject(); +} + +void MIDIMappingDialog::save_clicked() +{ +#if HAVE_CEF + // The native file dialog uses GTK+, which interferes with CEF's use of the GLib main loop. + QFileDialog::Options options(QFileDialog::DontUseNativeDialog); +#else + QFileDialog::Options options; +#endif + unique_ptr new_mapping = construct_mapping_proto_from_ui(); + QString filename = QFileDialog::getSaveFileName(this, + "Save MIDI mapping", QString(), tr("Mapping files (*.midimapping)"), /*selectedFilter=*/nullptr, options); + if (!filename.endsWith(".midimapping")) { + filename += ".midimapping"; + } + if (!save_midi_mapping_to_file(*new_mapping, filename.toStdString())) { + QMessageBox box; + box.setText("Could not save mapping to '" + filename + "'. Check that you have the right permissions and try again."); + box.exec(); + } +} + +void MIDIMappingDialog::load_clicked() +{ +#if HAVE_CEF + // The native file dialog uses GTK+, which interferes with CEF's use of the GLib main loop. + QFileDialog::Options options(QFileDialog::DontUseNativeDialog); +#else + QFileDialog::Options options; +#endif + QString filename = QFileDialog::getOpenFileName(this, + "Load MIDI mapping", QString(), tr("Mapping files (*.midimapping)"), /*selectedFilter=*/nullptr, options); + MIDIMappingProto new_mapping; + if (!load_midi_mapping_from_file(filename.toStdString(), &new_mapping)) { + QMessageBox box; + box.setText("Could not load mapping from '" + filename + "'. Check that the file exists, has the right permissions and is valid."); + box.exec(); + return; + } + + fill_controls_from_mapping(new_mapping); +} + +namespace { + +template +T *get_mutable_bus_message(MIDIMappingProto *mapping_proto, unsigned bus_idx, int field_number) +{ + while (size_t(mapping_proto->bus_mapping_size()) <= bus_idx) { + mapping_proto->add_bus_mapping(); + } + + MIDIMappingBusProto *bus_mapping = mapping_proto->mutable_bus_mapping(bus_idx); + const FieldDescriptor *descriptor = bus_mapping->GetDescriptor()->FindFieldByNumber(field_number); + const Reflection *bus_reflection = bus_mapping->GetReflection(); + return static_cast(bus_reflection->MutableMessage(bus_mapping, descriptor)); +} + +} // namespace + +unique_ptr MIDIMappingDialog::construct_mapping_proto_from_ui() +{ + unique_ptr mapping_proto(new MIDIMappingProto); + for (const InstantiatedSpinner &is : controller_spinners) { + const int val = is.spinner->value(); + if (val == -1) { + continue; + } + + MIDIControllerProto *controller_proto = + get_mutable_bus_message(mapping_proto.get(), is.bus_idx, is.field_number); + controller_proto->set_controller_number(val); + } + for (const InstantiatedSpinner &is : button_spinners) { + const int val = is.spinner->value(); + if (val == -1) { + continue; + } + + MIDIButtonProto *button_proto = + get_mutable_bus_message(mapping_proto.get(), is.bus_idx, is.field_number); + button_proto->set_note_number(val); + } + for (const InstantiatedSpinner &is : light_spinners) { + const int val = is.spinner->value(); + if (val == -1) { + continue; + } + + MIDILightProto *light_proto = + get_mutable_bus_message(mapping_proto.get(), is.bus_idx, is.field_number); + light_proto->set_note_number(val); + } + int highest_bank_used = 0; // 1-indexed. + for (const InstantiatedComboBox &ic : bank_combo_boxes) { + const int val = ic.combo_box->currentIndex(); + highest_bank_used = std::max(highest_bank_used, val); + if (val == 0) { + continue; + } + + const FieldDescriptor *descriptor = mapping_proto->GetDescriptor()->FindFieldByNumber(ic.field_number); + const Reflection *bus_reflection = mapping_proto->GetReflection(); + bus_reflection->SetInt32(mapping_proto.get(), descriptor, val - 1); + } + mapping_proto->set_num_controller_banks(highest_bank_used); + return mapping_proto; +} + +void MIDIMappingDialog::add_bank_selector(QTreeWidgetItem *item, const MIDIMappingProto &mapping_proto, int bank_field_number) +{ + if (bank_field_number == 0) { + return; + } + QComboBox *bank_selector = new QComboBox(this); + bank_selector->addItems(QStringList() << "" << "Bank 1" << "Bank 2" << "Bank 3" << "Bank 4" << "Bank 5"); + bank_selector->setAutoFillBackground(true); + + bank_combo_boxes.push_back(InstantiatedComboBox{ bank_selector, bank_field_number }); + + ui->treeWidget->setItemWidget(item, 1, bank_selector); +} + +void MIDIMappingDialog::add_controls(const string &heading, + MIDIMappingDialog::ControlType control_type, + MIDIMappingDialog::SpinnerGroup spinner_group, + const MIDIMappingProto &mapping_proto, + const vector &controls) +{ + QTreeWidgetItem *heading_item = new QTreeWidgetItem(ui->treeWidget); + heading_item->setText(0, QString::fromStdString(heading)); + heading_item->setFirstColumnSpanned(true); + heading_item->setExpanded(true); + for (const Control &control : controls) { + QTreeWidgetItem *item = new QTreeWidgetItem(heading_item); + heading_item->addChild(item); + add_bank_selector(item, mapping_proto, control.bank_field_number); + item->setText(0, QString::fromStdString(control.label + " ")); + + for (unsigned bus_idx = 0; bus_idx < num_buses; ++bus_idx) { + QSpinBox *spinner = new QSpinBox(this); + spinner->setRange(-1, 127); + spinner->setAutoFillBackground(true); + spinner->setSpecialValueText("\u200d"); // Zero-width joiner (ie., empty). + spinner->installEventFilter(this); // So we know when the focus changes. + ui->treeWidget->setItemWidget(item, bus_idx + 2, spinner); + + if (control_type == ControlType::CONTROLLER) { + controller_spinners.push_back(InstantiatedSpinner{ spinner, bus_idx, spinner_group, control.field_number }); + } else if (control_type == ControlType::BUTTON) { + button_spinners.push_back(InstantiatedSpinner{ spinner, bus_idx, spinner_group, control.field_number }); + } else { + assert(control_type == ControlType::LIGHT); + light_spinners.push_back(InstantiatedSpinner{ spinner, bus_idx, spinner_group, control.field_number }); + } + spinners[bus_idx][control.field_number] = SpinnerAndGroup{ spinner, spinner_group }; + connect(spinner, static_cast(&QSpinBox::valueChanged), + bind(&MIDIMappingDialog::update_guess_button_state, this)); + } + } + ui->treeWidget->addTopLevelItem(heading_item); +} + +void MIDIMappingDialog::fill_controls_from_mapping(const MIDIMappingProto &mapping_proto) +{ + for (const InstantiatedSpinner &is : controller_spinners) { + is.spinner->setValue(get_controller_mapping(mapping_proto, is.bus_idx, is.field_number, -1)); + } + for (const InstantiatedSpinner &is : button_spinners) { + is.spinner->setValue(get_button_mapping(mapping_proto, is.bus_idx, is.field_number, -1)); + } + for (const InstantiatedSpinner &is : light_spinners) { + is.spinner->setValue(get_light_mapping(mapping_proto, is.bus_idx, is.field_number, -1)); + } + for (const InstantiatedComboBox &ic : bank_combo_boxes) { + ic.combo_box->setCurrentIndex(get_bank(mapping_proto, ic.field_number, -1) + 1); + } +} + +void MIDIMappingDialog::controller_changed(unsigned controller) +{ + post_to_main_thread([=]{ + for (const InstantiatedSpinner &is : controller_spinners) { + if (is.spinner->hasFocus()) { + is.spinner->setValue(controller); + is.spinner->selectAll(); + } + } + }); +} + +void MIDIMappingDialog::note_on(unsigned note) +{ + post_to_main_thread([=]{ + for (const InstantiatedSpinner &is : button_spinners) { + if (is.spinner->hasFocus()) { + is.spinner->setValue(note); + is.spinner->selectAll(); + } + } + for (const InstantiatedSpinner &is : light_spinners) { + if (is.spinner->hasFocus()) { + is.spinner->setValue(note); + is.spinner->selectAll(); + } + } + }); +} + +pair MIDIMappingDialog::guess_offset(unsigned bus_idx, MIDIMappingDialog::SpinnerGroup spinner_group) +{ + constexpr pair not_found(-1, 0); + + if (bus_is_empty(bus_idx, spinner_group)) { + return not_found; + } + + // See if we can find a non-empty bus to source from (prefer from the left). + unsigned source_bus_idx; + if (bus_idx > 0 && !bus_is_empty(bus_idx - 1, spinner_group)) { + source_bus_idx = bus_idx - 1; + } else if (bus_idx < num_buses - 1 && !bus_is_empty(bus_idx + 1, spinner_group)) { + source_bus_idx = bus_idx + 1; + } else { + return not_found; + } + + // See if we can find a consistent offset. + bool found_offset = false; + int offset = 0; + int minimum_allowed_offset = numeric_limits::min(); + int maximum_allowed_offset = numeric_limits::max(); + for (const auto &field_number_and_spinner : spinners[bus_idx]) { + int field_number = field_number_and_spinner.first; + QSpinBox *spinner = field_number_and_spinner.second.spinner; + SpinnerGroup this_spinner_group = field_number_and_spinner.second.group; + assert(spinners[source_bus_idx].count(field_number)); + QSpinBox *source_spinner = spinners[source_bus_idx][field_number].spinner; + assert(spinners[source_bus_idx][field_number].group == this_spinner_group); + + if (spinner_group != SpinnerGroup::ALL_GROUPS && + spinner_group != this_spinner_group) { + continue; + } + if (spinner->value() == -1) { + if (source_spinner->value() != -1) { + // If the source value is e.g. 3, offset can't be less than -2 or larger than 124. + // Otherwise, we'd extrapolate values outside [1..127]. + minimum_allowed_offset = max(minimum_allowed_offset, 1 - source_spinner->value()); + maximum_allowed_offset = min(maximum_allowed_offset, 127 - source_spinner->value()); + } + continue; + } + if (source_spinner->value() == -1) { + // The bus has a controller set that the source bus doesn't set. + return not_found; + } + + int candidate_offset = spinner->value() - source_spinner->value(); + if (!found_offset) { + offset = candidate_offset; + found_offset = true; + } else if (candidate_offset != offset) { + return not_found; + } + } + + if (!found_offset) { + // Given that the bus wasn't empty, this shouldn't happen. + assert(false); + return not_found; + } + + if (offset < minimum_allowed_offset || offset > maximum_allowed_offset) { + return not_found; + } + return make_pair(source_bus_idx, offset); +} + +bool MIDIMappingDialog::bus_is_empty(unsigned bus_idx, SpinnerGroup spinner_group) +{ + for (const auto &field_number_and_spinner : spinners[bus_idx]) { + QSpinBox *spinner = field_number_and_spinner.second.spinner; + SpinnerGroup this_spinner_group = field_number_and_spinner.second.group; + if (spinner_group != SpinnerGroup::ALL_GROUPS && + spinner_group != this_spinner_group) { + continue; + } + if (spinner->value() != -1) { + return false; + } + } + return true; +} + +void MIDIMappingDialog::update_guess_button_state() +{ + FocusInfo focus = find_focus(); + if (focus.bus_idx < 0) { + return; + } + { + pair bus_and_offset = guess_offset(focus.bus_idx, SpinnerGroup::ALL_GROUPS); + ui->guess_bus_button->setEnabled(bus_and_offset.first != -1); + } + { + pair bus_and_offset = guess_offset(focus.bus_idx, focus.spinner_group); + ui->guess_group_button->setEnabled(bus_and_offset.first != -1); + } + last_focus = focus; +} + +MIDIMappingDialog::FocusInfo MIDIMappingDialog::find_focus() const +{ + for (const InstantiatedSpinner &is : controller_spinners) { + if (is.spinner->hasFocus()) { + return FocusInfo{ int(is.bus_idx), is.spinner_group, is.field_number }; + } + } + for (const InstantiatedSpinner &is : button_spinners) { + if (is.spinner->hasFocus()) { + return FocusInfo{ int(is.bus_idx), is.spinner_group, is.field_number }; + } + } + for (const InstantiatedSpinner &is : light_spinners) { + if (is.spinner->hasFocus()) { + return FocusInfo{ int(is.bus_idx), is.spinner_group, is.field_number }; + } + } + return FocusInfo{ -1, SpinnerGroup::ALL_GROUPS, -1 }; +} diff --git a/nageru/midi_mapping_dialog.h b/nageru/midi_mapping_dialog.h new file mode 100644 index 0000000..c36781d --- /dev/null +++ b/nageru/midi_mapping_dialog.h @@ -0,0 +1,170 @@ +#ifndef _MIDI_MAPPING_DIALOG_H +#define _MIDI_MAPPING_DIALOG_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "midi_mapper.h" + +class QEvent; +class QObject; + +namespace Ui { +class MIDIMappingDialog; +} // namespace Ui + +class MIDIMappingProto; +class QComboBox; +class QSpinBox; +class QTreeWidgetItem; + +class MIDIMappingDialog : public QDialog, public ControllerReceiver +{ + Q_OBJECT + +public: + MIDIMappingDialog(MIDIMapper *mapper); + ~MIDIMappingDialog(); + + bool eventFilter(QObject *obj, QEvent *event) override; + + // For use in midi_mapping_dialog.cpp only. + struct Control { + std::string label; + int field_number; // In MIDIMappingBusProto. + int bank_field_number; // In MIDIMappingProto. + }; + + // ControllerReceiver interface. We only implement the raw events. + // All values are [0.0, 1.0]. + void set_locut(float value) override {} + void set_limiter_threshold(float value) override {} + void set_makeup_gain(float value) override {} + + void set_stereo_width(unsigned bus_idx, float value) override {} + void set_treble(unsigned bus_idx, float value) override {} + void set_mid(unsigned bus_idx, float value) override {} + void set_bass(unsigned bus_idx, float value) override {} + void set_gain(unsigned bus_idx, float value) override {} + void set_compressor_threshold(unsigned bus_idx, float value) override {} + void set_fader(unsigned bus_idx, float value) override {} + + void toggle_mute(unsigned bus_idx) override {} + void toggle_locut(unsigned bus_idx) override {} + void toggle_auto_gain_staging(unsigned bus_idx) override {} + void toggle_compressor(unsigned bus_idx) override {} + void clear_peak(unsigned bus_idx) override {} + void toggle_limiter() override {} + void toggle_auto_makeup_gain() override {} + + void clear_all_highlights() override {} + + void highlight_locut(bool highlight) override {} + void highlight_limiter_threshold(bool highlight) override {} + void highlight_makeup_gain(bool highlight) override {} + + void highlight_stereo_width(unsigned bus_idx, bool highlight) override {} + void highlight_treble(unsigned bus_idx, bool highlight) override {} + void highlight_mid(unsigned bus_idx, bool highlight) override {} + void highlight_bass(unsigned bus_idx, bool highlight) override {} + void highlight_gain(unsigned bus_idx, bool highlight) override {} + void highlight_compressor_threshold(unsigned bus_idx, bool highlight) override {} + void highlight_fader(unsigned bus_idx, bool highlight) override {} + + void highlight_mute(unsigned bus_idx, bool highlight) override {} + void highlight_toggle_locut(unsigned bus_idx, bool highlight) override {} + void highlight_toggle_auto_gain_staging(unsigned bus_idx, bool highlight) override {} + void highlight_toggle_compressor(unsigned bus_idx, bool highlight) override {} + void highlight_clear_peak(unsigned bus_idx, bool highlight) override {} + void highlight_toggle_limiter(bool highlight) override {} + void highlight_toggle_auto_makeup_gain(bool highlight) override {} + + // Raw events; used for the editor dialog only. + void controller_changed(unsigned controller) override; + void note_on(unsigned note) override; + +public slots: + void guess_clicked(bool limit_to_group); + void ok_clicked(); + void cancel_clicked(); + void save_clicked(); + void load_clicked(); + +private: + static constexpr unsigned num_buses = 8; + + // Each spinner belongs to exactly one group, corresponding to the + // subheadings in the UI. This is so that we can extrapolate data + // across only single groups if need be. + enum class SpinnerGroup { + ALL_GROUPS = -1, + PER_BUS_CONTROLLERS, + PER_BUS_BUTTONS, + PER_BUS_LIGHTS, + GLOBAL_CONTROLLERS, + GLOBAL_BUTTONS, + GLOBAL_LIGHTS + }; + + void add_bank_selector(QTreeWidgetItem *item, const MIDIMappingProto &mapping_proto, int bank_field_number); + + enum class ControlType { CONTROLLER, BUTTON, LIGHT }; + void add_controls(const std::string &heading, ControlType control_type, + SpinnerGroup spinner_group, + const MIDIMappingProto &mapping_proto, const std::vector &controls); + void fill_controls_from_mapping(const MIDIMappingProto &mapping_proto); + + // Tries to find a source bus and an offset to it that would give + // a consistent offset for the rest of the mappings in this bus. + // Returns -1 for the bus if no consistent offset can be found. + std::pair guess_offset(unsigned bus_idx, SpinnerGroup spinner_group); + bool bus_is_empty(unsigned bus_idx, SpinnerGroup spinner_group); + + void update_guess_button_state(); + struct FocusInfo { + int bus_idx; // -1 for none. + SpinnerGroup spinner_group; + int field_number; + }; + FocusInfo find_focus() const; + + std::unique_ptr construct_mapping_proto_from_ui(); + + Ui::MIDIMappingDialog *ui; + MIDIMapper *mapper; + ControllerReceiver *old_receiver; + FocusInfo last_focus{-1, SpinnerGroup::ALL_GROUPS, -1}; + + // All controllers actually laid out on the grid (we need to store them + // so that we can move values back and forth between the controls and + // the protobuf on save/load). + struct InstantiatedSpinner { + QSpinBox *spinner; + unsigned bus_idx; + SpinnerGroup spinner_group; + int field_number; // In MIDIMappingBusProto. + }; + struct InstantiatedComboBox { + QComboBox *combo_box; + int field_number; // In MIDIMappingProto. + }; + std::vector controller_spinners; + std::vector button_spinners; + std::vector light_spinners; + std::vector bank_combo_boxes; + + // Keyed on bus index, then field number. + struct SpinnerAndGroup { + QSpinBox *spinner; + SpinnerGroup group; + }; + std::map> spinners; +}; + +#endif // !defined(_MIDI_MAPPING_DIALOG_H) diff --git a/nageru/mixer.cpp b/nageru/mixer.cpp new file mode 100644 index 0000000..deaa8e7 --- /dev/null +++ b/nageru/mixer.cpp @@ -0,0 +1,1734 @@ +#undef Success + +#include "mixer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "DeckLinkAPI.h" +#include "LinuxCOM.h" +#include "alsa_output.h" +#include "basic_stats.h" +#include "bmusb/bmusb.h" +#include "bmusb/fake_capture.h" +#ifdef HAVE_CEF +#include "cef_capture.h" +#endif +#include "chroma_subsampler.h" +#include "context.h" +#include "decklink_capture.h" +#include "decklink_output.h" +#include "defs.h" +#include "disk_space_estimator.h" +#include "ffmpeg_capture.h" +#include "flags.h" +#include "input_mapping.h" +#include "metrics.h" +#include "pbo_frame_allocator.h" +#include "ref_counted_gl_sync.h" +#include "resampling_queue.h" +#include "timebase.h" +#include "timecode_renderer.h" +#include "v210_converter.h" +#include "video_encoder.h" + +#undef Status +#include +#include "json.pb.h" + +class IDeckLink; +class QOpenGLContext; + +using namespace movit; +using namespace std; +using namespace std::chrono; +using namespace std::placeholders; +using namespace bmusb; + +Mixer *global_mixer = nullptr; + +namespace { + +void insert_new_frame(RefCountedFrame frame, unsigned field_num, bool interlaced, unsigned card_index, InputState *input_state) +{ + if (interlaced) { + for (unsigned frame_num = FRAME_HISTORY_LENGTH; frame_num --> 1; ) { // :-) + input_state->buffered_frames[card_index][frame_num] = + input_state->buffered_frames[card_index][frame_num - 1]; + } + input_state->buffered_frames[card_index][0] = { frame, field_num }; + } else { + for (unsigned frame_num = 0; frame_num < FRAME_HISTORY_LENGTH; ++frame_num) { + input_state->buffered_frames[card_index][frame_num] = { frame, field_num }; + } + } +} + +void ensure_texture_resolution(PBOFrameAllocator::Userdata *userdata, unsigned field, unsigned width, unsigned height, unsigned cbcr_width, unsigned cbcr_height, unsigned v210_width) +{ + bool first; + switch (userdata->pixel_format) { + case PixelFormat_10BitYCbCr: + first = userdata->tex_v210[field] == 0 || userdata->tex_444[field] == 0; + break; + case PixelFormat_8BitYCbCr: + first = userdata->tex_y[field] == 0 || userdata->tex_cbcr[field] == 0; + break; + case PixelFormat_8BitBGRA: + first = userdata->tex_rgba[field] == 0; + break; + case PixelFormat_8BitYCbCrPlanar: + first = userdata->tex_y[field] == 0 || userdata->tex_cb[field] == 0 || userdata->tex_cr[field] == 0; + break; + default: + assert(false); + } + + if (first || + width != userdata->last_width[field] || + height != userdata->last_height[field] || + cbcr_width != userdata->last_cbcr_width[field] || + cbcr_height != userdata->last_cbcr_height[field]) { + // We changed resolution since last use of this texture, so we need to create + // a new object. Note that this each card has its own PBOFrameAllocator, + // we don't need to worry about these flip-flopping between resolutions. + switch (userdata->pixel_format) { + case PixelFormat_10BitYCbCr: + glBindTexture(GL_TEXTURE_2D, userdata->tex_444[field]); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB10_A2, width, height, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, nullptr); + check_error(); + break; + case PixelFormat_8BitYCbCr: { + glBindTexture(GL_TEXTURE_2D, userdata->tex_cbcr[field]); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, cbcr_width, height, 0, GL_RG, GL_UNSIGNED_BYTE, nullptr); + check_error(); + glBindTexture(GL_TEXTURE_2D, userdata->tex_y[field]); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + break; + } + case PixelFormat_8BitYCbCrPlanar: { + glBindTexture(GL_TEXTURE_2D, userdata->tex_y[field]); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + glBindTexture(GL_TEXTURE_2D, userdata->tex_cb[field]); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, cbcr_width, cbcr_height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + glBindTexture(GL_TEXTURE_2D, userdata->tex_cr[field]); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, cbcr_width, cbcr_height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + break; + } + case PixelFormat_8BitBGRA: + glBindTexture(GL_TEXTURE_2D, userdata->tex_rgba[field]); + check_error(); + if (global_flags.can_disable_srgb_decoder) { // See the comments in tweaked_inputs.h. + glTexImage2D(GL_TEXTURE_2D, 0, GL_SRGB8_ALPHA8, width, height, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, nullptr); + } else { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, nullptr); + } + check_error(); + break; + default: + assert(false); + } + userdata->last_width[field] = width; + userdata->last_height[field] = height; + userdata->last_cbcr_width[field] = cbcr_width; + userdata->last_cbcr_height[field] = cbcr_height; + } + if (global_flags.ten_bit_input && + (first || v210_width != userdata->last_v210_width[field])) { + // Same as above; we need to recreate the texture. + glBindTexture(GL_TEXTURE_2D, userdata->tex_v210[field]); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB10_A2, v210_width, height, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, nullptr); + check_error(); + userdata->last_v210_width[field] = v210_width; + } +} + +void upload_texture(GLuint tex, GLuint width, GLuint height, GLuint stride, bool interlaced_stride, GLenum format, GLenum type, GLintptr offset) +{ + if (interlaced_stride) { + stride *= 2; + } + if (global_flags.flush_pbos) { + glFlushMappedBufferRange(GL_PIXEL_UNPACK_BUFFER, offset, stride * height); + check_error(); + } + + glBindTexture(GL_TEXTURE_2D, tex); + check_error(); + if (interlaced_stride) { + glPixelStorei(GL_UNPACK_ROW_LENGTH, width * 2); + check_error(); + } else { + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + check_error(); + } + + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, format, type, BUFFER_OFFSET(offset)); + check_error(); + glBindTexture(GL_TEXTURE_2D, 0); + check_error(); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + check_error(); +} + +} // namespace + +void JitterHistory::register_metrics(const vector> &labels) +{ + global_metrics.add("input_underestimated_jitter_frames", labels, &metric_input_underestimated_jitter_frames); + global_metrics.add("input_estimated_max_jitter_seconds", labels, &metric_input_estimated_max_jitter_seconds, Metrics::TYPE_GAUGE); +} + +void JitterHistory::unregister_metrics(const vector> &labels) +{ + global_metrics.remove("input_underestimated_jitter_frames", labels); + global_metrics.remove("input_estimated_max_jitter_seconds", labels); +} + +void JitterHistory::frame_arrived(steady_clock::time_point now, int64_t frame_duration, size_t dropped_frames) +{ + if (expected_timestamp > steady_clock::time_point::min()) { + expected_timestamp += dropped_frames * nanoseconds(frame_duration * 1000000000 / TIMEBASE); + double jitter_seconds = fabs(duration(expected_timestamp - now).count()); + history.push_back(orders.insert(jitter_seconds)); + if (jitter_seconds > estimate_max_jitter()) { + ++metric_input_underestimated_jitter_frames; + } + + metric_input_estimated_max_jitter_seconds = estimate_max_jitter(); + + if (history.size() > history_length) { + orders.erase(history.front()); + history.pop_front(); + } + assert(history.size() <= history_length); + } + expected_timestamp = now + nanoseconds(frame_duration * 1000000000 / TIMEBASE); +} + +double JitterHistory::estimate_max_jitter() const +{ + if (orders.empty()) { + return 0.0; + } + size_t elem_idx = lrint((orders.size() - 1) * percentile); + if (percentile <= 0.5) { + return *next(orders.begin(), elem_idx) * multiplier; + } else { + return *prev(orders.end(), orders.size() - elem_idx) * multiplier; + } +} + +void QueueLengthPolicy::register_metrics(const vector> &labels) +{ + global_metrics.add("input_queue_safe_length_frames", labels, &metric_input_queue_safe_length_frames, Metrics::TYPE_GAUGE); +} + +void QueueLengthPolicy::unregister_metrics(const vector> &labels) +{ + global_metrics.remove("input_queue_safe_length_frames", labels); +} + +void QueueLengthPolicy::update_policy(steady_clock::time_point now, + steady_clock::time_point expected_next_frame, + int64_t input_frame_duration, + int64_t master_frame_duration, + double max_input_card_jitter_seconds, + double max_master_card_jitter_seconds) +{ + double input_frame_duration_seconds = input_frame_duration / double(TIMEBASE); + double master_frame_duration_seconds = master_frame_duration / double(TIMEBASE); + + // Figure out when we can expect the next frame for this card, assuming + // worst-case jitter (ie., the frame is maximally late). + double seconds_until_next_frame = max(duration(expected_next_frame - now).count() + max_input_card_jitter_seconds, 0.0); + + // How many times are the master card expected to tick in that time? + // We assume the master clock has worst-case jitter but not any rate + // discrepancy, ie., it ticks as early as possible every time, but not + // cumulatively. + double frames_needed = (seconds_until_next_frame + max_master_card_jitter_seconds) / master_frame_duration_seconds; + + // As a special case, if the master card ticks faster than the input card, + // we expect the queue to drain by itself even without dropping. But if + // the difference is small (e.g. 60 Hz master and 59.94 input), it would + // go slowly enough that the effect wouldn't really be appreciable. + // We account for this by looking at the situation five frames ahead, + // assuming everything else is the same. + double frames_allowed; + if (master_frame_duration < input_frame_duration) { + frames_allowed = frames_needed + 5 * (input_frame_duration_seconds - master_frame_duration_seconds) / master_frame_duration_seconds; + } else { + frames_allowed = frames_needed; + } + + safe_queue_length = max(floor(frames_allowed), 0); + metric_input_queue_safe_length_frames = safe_queue_length; +} + +Mixer::Mixer(const QSurfaceFormat &format, unsigned num_cards) + : httpd(), + num_cards(num_cards), + mixer_surface(create_surface(format)), + h264_encoder_surface(create_surface(format)), + decklink_output_surface(create_surface(format)) +{ + memcpy(ycbcr_interpretation, global_flags.ycbcr_interpretation, sizeof(ycbcr_interpretation)); + CHECK(init_movit(MOVIT_SHADER_DIR, MOVIT_DEBUG_OFF)); + check_error(); + + // This nearly always should be true. + global_flags.can_disable_srgb_decoder = + epoxy_has_gl_extension("GL_EXT_texture_sRGB_decode") && + epoxy_has_gl_extension("GL_ARB_sampler_objects"); + + // Since we allow non-bouncing 4:2:2 YCbCrInputs, effective subpixel precision + // will be halved when sampling them, and we need to compensate here. + movit_texel_subpixel_precision /= 2.0; + + resource_pool.reset(new ResourcePool); + for (unsigned i = 0; i < NUM_OUTPUTS; ++i) { + output_channel[i].parent = this; + output_channel[i].channel = i; + } + + ImageFormat inout_format; + inout_format.color_space = COLORSPACE_sRGB; + inout_format.gamma_curve = GAMMA_sRGB; + + // Matches the 4:2:0 format created by the main chain. + YCbCrFormat ycbcr_format; + ycbcr_format.chroma_subsampling_x = 2; + ycbcr_format.chroma_subsampling_y = 2; + if (global_flags.ycbcr_rec709_coefficients) { + ycbcr_format.luma_coefficients = YCBCR_REC_709; + } else { + ycbcr_format.luma_coefficients = YCBCR_REC_601; + } + ycbcr_format.full_range = false; + ycbcr_format.num_levels = 1 << global_flags.x264_bit_depth; + ycbcr_format.cb_x_position = 0.0f; + ycbcr_format.cr_x_position = 0.0f; + ycbcr_format.cb_y_position = 0.5f; + ycbcr_format.cr_y_position = 0.5f; + + // Display chain; shows the live output produced by the main chain (or rather, a copy of it). + display_chain.reset(new EffectChain(global_flags.width, global_flags.height, resource_pool.get())); + check_error(); + GLenum type = global_flags.x264_bit_depth > 8 ? GL_UNSIGNED_SHORT : GL_UNSIGNED_BYTE; + display_input = new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height, YCBCR_INPUT_SPLIT_Y_AND_CBCR, type); + display_chain->add_input(display_input); + display_chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED); + display_chain->set_dither_bits(0); // Don't bother. + display_chain->finalize(); + + video_encoder.reset(new VideoEncoder(resource_pool.get(), h264_encoder_surface, global_flags.va_display, global_flags.width, global_flags.height, &httpd, global_disk_space_estimator)); + + // Must be instantiated after VideoEncoder has initialized global_flags.use_zerocopy. + theme.reset(new Theme(global_flags.theme_filename, global_flags.theme_dirs, resource_pool.get(), num_cards)); + + // Must be instantiated after the theme, as the theme decides the number of FFmpeg inputs. + std::vector video_inputs = theme->get_video_inputs(); + audio_mixer.reset(new AudioMixer(num_cards, video_inputs.size())); + + httpd.add_endpoint("/channels", bind(&Mixer::get_channels_json, this), HTTPD::ALLOW_ALL_ORIGINS); + for (int channel_idx = 2; channel_idx < theme->get_num_channels(); ++channel_idx) { + char url[256]; + snprintf(url, sizeof(url), "/channels/%d/color", channel_idx); + httpd.add_endpoint(url, bind(&Mixer::get_channel_color_http, this, unsigned(channel_idx)), HTTPD::ALLOW_ALL_ORIGINS); + } + + // Start listening for clients only once VideoEncoder has written its header, if any. + httpd.start(global_flags.http_port); + + // First try initializing the then PCI devices, then USB, then + // fill up with fake cards until we have the desired number of cards. + unsigned num_pci_devices = 0; + unsigned card_index = 0; + + { + IDeckLinkIterator *decklink_iterator = CreateDeckLinkIteratorInstance(); + if (decklink_iterator != nullptr) { + for ( ; card_index < num_cards; ++card_index) { + IDeckLink *decklink; + if (decklink_iterator->Next(&decklink) != S_OK) { + break; + } + + DeckLinkCapture *capture = new DeckLinkCapture(decklink, card_index); + DeckLinkOutput *output = new DeckLinkOutput(resource_pool.get(), decklink_output_surface, global_flags.width, global_flags.height, card_index); + if (!output->set_device(decklink)) { + delete output; + output = nullptr; + } + configure_card(card_index, capture, CardType::LIVE_CARD, output); + ++num_pci_devices; + } + decklink_iterator->Release(); + fprintf(stderr, "Found %u DeckLink PCI card(s).\n", num_pci_devices); + } else { + fprintf(stderr, "DeckLink drivers not found. Probing for USB cards only.\n"); + } + } + + unsigned num_usb_devices = BMUSBCapture::num_cards(); + for (unsigned usb_card_index = 0; usb_card_index < num_usb_devices && card_index < num_cards; ++usb_card_index, ++card_index) { + BMUSBCapture *capture = new BMUSBCapture(usb_card_index); + capture->set_card_disconnected_callback(bind(&Mixer::bm_hotplug_remove, this, card_index)); + configure_card(card_index, capture, CardType::LIVE_CARD, /*output=*/nullptr); + } + fprintf(stderr, "Found %u USB card(s).\n", num_usb_devices); + + unsigned num_fake_cards = 0; + for ( ; card_index < num_cards; ++card_index, ++num_fake_cards) { + FakeCapture *capture = new FakeCapture(global_flags.width, global_flags.height, FAKE_FPS, OUTPUT_FREQUENCY, card_index, global_flags.fake_cards_audio); + configure_card(card_index, capture, CardType::FAKE_CAPTURE, /*output=*/nullptr); + } + + if (num_fake_cards > 0) { + fprintf(stderr, "Initialized %u fake cards.\n", num_fake_cards); + } + + // Initialize all video inputs the theme asked for. Note that these are + // all put _after_ the regular cards, which stop at - 1. + for (unsigned video_card_index = 0; video_card_index < video_inputs.size(); ++card_index, ++video_card_index) { + if (card_index >= MAX_VIDEO_CARDS) { + fprintf(stderr, "ERROR: Not enough card slots available for the videos the theme requested.\n"); + exit(1); + } + configure_card(card_index, video_inputs[video_card_index], CardType::FFMPEG_INPUT, /*output=*/nullptr); + video_inputs[video_card_index]->set_card_index(card_index); + } + num_video_inputs = video_inputs.size(); + +#ifdef HAVE_CEF + // Same, for HTML inputs. + std::vector html_inputs = theme->get_html_inputs(); + for (unsigned html_card_index = 0; html_card_index < html_inputs.size(); ++card_index, ++html_card_index) { + if (card_index >= MAX_VIDEO_CARDS) { + fprintf(stderr, "ERROR: Not enough card slots available for the HTML inputs the theme requested.\n"); + exit(1); + } + configure_card(card_index, html_inputs[html_card_index], CardType::CEF_INPUT, /*output=*/nullptr); + html_inputs[html_card_index]->set_card_index(card_index); + } + num_html_inputs = html_inputs.size(); +#endif + + BMUSBCapture::set_card_connected_callback(bind(&Mixer::bm_hotplug_add, this, _1)); + BMUSBCapture::start_bm_thread(); + + for (unsigned card_index = 0; card_index < num_cards + num_video_inputs + num_html_inputs; ++card_index) { + cards[card_index].queue_length_policy.reset(card_index); + } + + chroma_subsampler.reset(new ChromaSubsampler(resource_pool.get())); + + if (global_flags.ten_bit_input) { + if (!v210Converter::has_hardware_support()) { + fprintf(stderr, "ERROR: --ten-bit-input requires support for OpenGL compute shaders\n"); + fprintf(stderr, " (OpenGL 4.3, or GL_ARB_compute_shader + GL_ARB_shader_image_load_store).\n"); + exit(1); + } + v210_converter.reset(new v210Converter()); + + // These are all the widths listed in the Blackmagic SDK documentation + // (section 2.7.3, “Display Modes”). + v210_converter->precompile_shader(720); + v210_converter->precompile_shader(1280); + v210_converter->precompile_shader(1920); + v210_converter->precompile_shader(2048); + v210_converter->precompile_shader(3840); + v210_converter->precompile_shader(4096); + } + if (global_flags.ten_bit_output) { + if (!v210Converter::has_hardware_support()) { + fprintf(stderr, "ERROR: --ten-bit-output requires support for OpenGL compute shaders\n"); + fprintf(stderr, " (OpenGL 4.3, or GL_ARB_compute_shader + GL_ARB_shader_image_load_store).\n"); + exit(1); + } + } + + timecode_renderer.reset(new TimecodeRenderer(resource_pool.get(), global_flags.width, global_flags.height)); + display_timecode_in_stream = global_flags.display_timecode_in_stream; + display_timecode_on_stdout = global_flags.display_timecode_on_stdout; + + if (global_flags.enable_alsa_output) { + alsa.reset(new ALSAOutput(OUTPUT_FREQUENCY, /*num_channels=*/2)); + } + if (global_flags.output_card != -1) { + desired_output_card_index = global_flags.output_card; + set_output_card_internal(global_flags.output_card); + } + + output_jitter_history.register_metrics({{ "card", "output" }}); +} + +Mixer::~Mixer() +{ + httpd.stop(); + BMUSBCapture::stop_bm_thread(); + + for (unsigned card_index = 0; card_index < num_cards + num_video_inputs + num_html_inputs; ++card_index) { + cards[card_index].capture->stop_dequeue_thread(); + if (cards[card_index].output) { + cards[card_index].output->end_output(); + cards[card_index].output.reset(); + } + } + + video_encoder.reset(nullptr); +} + +void Mixer::configure_card(unsigned card_index, CaptureInterface *capture, CardType card_type, DeckLinkOutput *output) +{ + printf("Configuring card %d...\n", card_index); + + CaptureCard *card = &cards[card_index]; + if (card->capture != nullptr) { + card->capture->stop_dequeue_thread(); + } + card->capture.reset(capture); + card->is_fake_capture = (card_type == CardType::FAKE_CAPTURE); + card->is_cef_capture = (card_type == CardType::CEF_INPUT); + card->may_have_dropped_last_frame = false; + card->type = card_type; + if (card->output.get() != output) { + card->output.reset(output); + } + + PixelFormat pixel_format; + if (card_type == CardType::FFMPEG_INPUT) { + pixel_format = capture->get_current_pixel_format(); + } else if (card_type == CardType::CEF_INPUT) { + pixel_format = PixelFormat_8BitBGRA; + } else if (global_flags.ten_bit_input) { + pixel_format = PixelFormat_10BitYCbCr; + } else { + pixel_format = PixelFormat_8BitYCbCr; + } + + card->capture->set_frame_callback(bind(&Mixer::bm_frame, this, card_index, _1, _2, _3, _4, _5, _6, _7)); + if (card->frame_allocator == nullptr) { + card->frame_allocator.reset(new PBOFrameAllocator(pixel_format, 8 << 20, global_flags.width, global_flags.height)); // 8 MB. + } + card->capture->set_video_frame_allocator(card->frame_allocator.get()); + if (card->surface == nullptr) { + card->surface = create_surface_with_same_format(mixer_surface); + } + while (!card->new_frames.empty()) card->new_frames.pop_front(); + card->last_timecode = -1; + card->capture->set_pixel_format(pixel_format); + card->capture->configure_card(); + + // NOTE: start_bm_capture() happens in thread_func(). + + DeviceSpec device; + if (card_type == CardType::FFMPEG_INPUT) { + device = DeviceSpec{InputSourceType::FFMPEG_VIDEO_INPUT, card_index - num_cards}; + } else { + device = DeviceSpec{InputSourceType::CAPTURE_CARD, card_index}; + } + audio_mixer->reset_resampler(device); + audio_mixer->set_display_name(device, card->capture->get_description()); + audio_mixer->trigger_state_changed_callback(); + + // Unregister old metrics, if any. + if (!card->labels.empty()) { + const vector> &labels = card->labels; + card->jitter_history.unregister_metrics(labels); + card->queue_length_policy.unregister_metrics(labels); + global_metrics.remove("input_received_frames", labels); + global_metrics.remove("input_dropped_frames_jitter", labels); + global_metrics.remove("input_dropped_frames_error", labels); + global_metrics.remove("input_dropped_frames_resets", labels); + global_metrics.remove("input_queue_length_frames", labels); + global_metrics.remove("input_queue_duped_frames", labels); + + global_metrics.remove("input_has_signal_bool", labels); + global_metrics.remove("input_is_connected_bool", labels); + global_metrics.remove("input_interlaced_bool", labels); + global_metrics.remove("input_width_pixels", labels); + global_metrics.remove("input_height_pixels", labels); + global_metrics.remove("input_frame_rate_nom", labels); + global_metrics.remove("input_frame_rate_den", labels); + global_metrics.remove("input_sample_rate_hz", labels); + } + + // Register metrics. + vector> labels; + char card_name[64]; + snprintf(card_name, sizeof(card_name), "%d", card_index); + labels.emplace_back("card", card_name); + + switch (card_type) { + case CardType::LIVE_CARD: + labels.emplace_back("cardtype", "live"); + break; + case CardType::FAKE_CAPTURE: + labels.emplace_back("cardtype", "fake"); + break; + case CardType::FFMPEG_INPUT: + labels.emplace_back("cardtype", "ffmpeg"); + break; + case CardType::CEF_INPUT: + labels.emplace_back("cardtype", "cef"); + break; + default: + assert(false); + } + card->jitter_history.register_metrics(labels); + card->queue_length_policy.register_metrics(labels); + global_metrics.add("input_received_frames", labels, &card->metric_input_received_frames); + global_metrics.add("input_dropped_frames_jitter", labels, &card->metric_input_dropped_frames_jitter); + global_metrics.add("input_dropped_frames_error", labels, &card->metric_input_dropped_frames_error); + global_metrics.add("input_dropped_frames_resets", labels, &card->metric_input_resets); + global_metrics.add("input_queue_length_frames", labels, &card->metric_input_queue_length_frames, Metrics::TYPE_GAUGE); + global_metrics.add("input_queue_duped_frames", labels, &card->metric_input_duped_frames); + + global_metrics.add("input_has_signal_bool", labels, &card->metric_input_has_signal_bool, Metrics::TYPE_GAUGE); + global_metrics.add("input_is_connected_bool", labels, &card->metric_input_is_connected_bool, Metrics::TYPE_GAUGE); + global_metrics.add("input_interlaced_bool", labels, &card->metric_input_interlaced_bool, Metrics::TYPE_GAUGE); + global_metrics.add("input_width_pixels", labels, &card->metric_input_width_pixels, Metrics::TYPE_GAUGE); + global_metrics.add("input_height_pixels", labels, &card->metric_input_height_pixels, Metrics::TYPE_GAUGE); + global_metrics.add("input_frame_rate_nom", labels, &card->metric_input_frame_rate_nom, Metrics::TYPE_GAUGE); + global_metrics.add("input_frame_rate_den", labels, &card->metric_input_frame_rate_den, Metrics::TYPE_GAUGE); + global_metrics.add("input_sample_rate_hz", labels, &card->metric_input_sample_rate_hz, Metrics::TYPE_GAUGE); + card->labels = labels; +} + +void Mixer::set_output_card_internal(int card_index) +{ + // We don't really need to take card_mutex, since we're in the mixer + // thread and don't mess with any queues (which is the only thing that happens + // from other threads), but it's probably the safest in the long run. + unique_lock lock(card_mutex); + if (output_card_index != -1) { + // Switch the old card from output to input. + CaptureCard *old_card = &cards[output_card_index]; + old_card->output->end_output(); + + // Stop the fake card that we put into place. + // This needs to _not_ happen under the mutex, to avoid deadlock + // (delivering the last frame needs to take the mutex). + CaptureInterface *fake_capture = old_card->capture.get(); + lock.unlock(); + fake_capture->stop_dequeue_thread(); + lock.lock(); + old_card->capture = move(old_card->parked_capture); // TODO: reset the metrics + old_card->is_fake_capture = false; + old_card->capture->start_bm_capture(); + } + if (card_index != -1) { + CaptureCard *card = &cards[card_index]; + CaptureInterface *capture = card->capture.get(); + // TODO: DeckLinkCapture::stop_dequeue_thread can actually take + // several seconds to complete (blocking on DisableVideoInput); + // see if we can maybe do it asynchronously. + lock.unlock(); + capture->stop_dequeue_thread(); + lock.lock(); + card->parked_capture = move(card->capture); + CaptureInterface *fake_capture = new FakeCapture(global_flags.width, global_flags.height, FAKE_FPS, OUTPUT_FREQUENCY, card_index, global_flags.fake_cards_audio); + configure_card(card_index, fake_capture, CardType::FAKE_CAPTURE, card->output.release()); + card->queue_length_policy.reset(card_index); + card->capture->start_bm_capture(); + desired_output_video_mode = output_video_mode = card->output->pick_video_mode(desired_output_video_mode); + card->output->start_output(desired_output_video_mode, pts_int); + } + output_card_index = card_index; + output_jitter_history.clear(); +} + +namespace { + +int unwrap_timecode(uint16_t current_wrapped, int last) +{ + uint16_t last_wrapped = last & 0xffff; + if (current_wrapped > last_wrapped) { + return (last & ~0xffff) | current_wrapped; + } else { + return 0x10000 + ((last & ~0xffff) | current_wrapped); + } +} + +DeviceSpec card_index_to_device(unsigned card_index, unsigned num_cards) +{ + if (card_index >= num_cards) { + return DeviceSpec{InputSourceType::FFMPEG_VIDEO_INPUT, card_index - num_cards}; + } else { + return DeviceSpec{InputSourceType::CAPTURE_CARD, card_index}; + } +} + +} // namespace + +void Mixer::bm_frame(unsigned card_index, uint16_t timecode, + FrameAllocator::Frame video_frame, size_t video_offset, VideoFormat video_format, + FrameAllocator::Frame audio_frame, size_t audio_offset, AudioFormat audio_format) +{ + DeviceSpec device = card_index_to_device(card_index, num_cards); + CaptureCard *card = &cards[card_index]; + + ++card->metric_input_received_frames; + card->metric_input_has_signal_bool = video_format.has_signal; + card->metric_input_is_connected_bool = video_format.is_connected; + card->metric_input_interlaced_bool = video_format.interlaced; + card->metric_input_width_pixels = video_format.width; + card->metric_input_height_pixels = video_format.height; + card->metric_input_frame_rate_nom = video_format.frame_rate_nom; + card->metric_input_frame_rate_den = video_format.frame_rate_den; + card->metric_input_sample_rate_hz = audio_format.sample_rate; + + if (is_mode_scanning[card_index]) { + if (video_format.has_signal) { + // Found a stable signal, so stop scanning. + is_mode_scanning[card_index] = false; + } else { + static constexpr double switch_time_s = 0.1; // Should be enough time for the signal to stabilize. + steady_clock::time_point now = steady_clock::now(); + double sec_since_last_switch = duration(steady_clock::now() - last_mode_scan_change[card_index]).count(); + if (sec_since_last_switch > switch_time_s) { + // It isn't this mode; try the next one. + mode_scanlist_index[card_index]++; + mode_scanlist_index[card_index] %= mode_scanlist[card_index].size(); + cards[card_index].capture->set_video_mode(mode_scanlist[card_index][mode_scanlist_index[card_index]]); + last_mode_scan_change[card_index] = now; + } + } + } + + int64_t frame_length = int64_t(TIMEBASE) * video_format.frame_rate_den / video_format.frame_rate_nom; + assert(frame_length > 0); + + size_t num_samples = (audio_frame.len > audio_offset) ? (audio_frame.len - audio_offset) / audio_format.num_channels / (audio_format.bits_per_sample / 8) : 0; + if (num_samples > OUTPUT_FREQUENCY / 10 && card->type != CardType::FFMPEG_INPUT) { + printf("%s: Dropping frame with implausible audio length (len=%d, offset=%d) [timecode=0x%04x video_len=%d video_offset=%d video_format=%x)\n", + spec_to_string(device).c_str(), int(audio_frame.len), int(audio_offset), + timecode, int(video_frame.len), int(video_offset), video_format.id); + if (video_frame.owner) { + video_frame.owner->release_frame(video_frame); + } + if (audio_frame.owner) { + audio_frame.owner->release_frame(audio_frame); + } + return; + } + + int dropped_frames = 0; + if (card->last_timecode != -1) { + dropped_frames = unwrap_timecode(timecode, card->last_timecode) - card->last_timecode - 1; + } + + // Number of samples per frame if we need to insert silence. + // (Could be nonintegral, but resampling will save us then.) + const int silence_samples = OUTPUT_FREQUENCY * video_format.frame_rate_den / video_format.frame_rate_nom; + + if (dropped_frames > MAX_FPS * 2) { + fprintf(stderr, "%s lost more than two seconds (or time code jumping around; from 0x%04x to 0x%04x), resetting resampler\n", + spec_to_string(device).c_str(), card->last_timecode, timecode); + audio_mixer->reset_resampler(device); + dropped_frames = 0; + ++card->metric_input_resets; + } else if (dropped_frames > 0) { + // Insert silence as needed. + fprintf(stderr, "%s dropped %d frame(s) (before timecode 0x%04x), inserting silence.\n", + spec_to_string(device).c_str(), dropped_frames, timecode); + card->metric_input_dropped_frames_error += dropped_frames; + + bool success; + do { + success = audio_mixer->add_silence(device, silence_samples, dropped_frames, frame_length); + } while (!success); + } + + if (num_samples > 0) { + audio_mixer->add_audio(device, audio_frame.data + audio_offset, num_samples, audio_format, frame_length, audio_frame.received_timestamp); + } + + // Done with the audio, so release it. + if (audio_frame.owner) { + audio_frame.owner->release_frame(audio_frame); + } + + card->last_timecode = timecode; + + PBOFrameAllocator::Userdata *userdata = (PBOFrameAllocator::Userdata *)video_frame.userdata; + + size_t cbcr_width, cbcr_height, cbcr_offset, y_offset; + size_t expected_length = video_format.stride * (video_format.height + video_format.extra_lines_top + video_format.extra_lines_bottom); + if (userdata != nullptr && userdata->pixel_format == PixelFormat_8BitYCbCrPlanar) { + // The calculation above is wrong for planar Y'CbCr, so just override it. + assert(card->type == CardType::FFMPEG_INPUT); + assert(video_offset == 0); + expected_length = video_frame.len; + + userdata->ycbcr_format = (static_cast(card->capture.get()))->get_current_frame_ycbcr_format(); + cbcr_width = video_format.width / userdata->ycbcr_format.chroma_subsampling_x; + cbcr_height = video_format.height / userdata->ycbcr_format.chroma_subsampling_y; + cbcr_offset = video_format.width * video_format.height; + y_offset = 0; + } else { + // All the other Y'CbCr formats are 4:2:2. + cbcr_width = video_format.width / 2; + cbcr_height = video_format.height; + cbcr_offset = video_offset / 2; + y_offset = video_frame.size / 2 + video_offset / 2; + } + if (video_frame.len - video_offset == 0 || + video_frame.len - video_offset != expected_length) { + if (video_frame.len != 0) { + printf("%s: Dropping video frame with wrong length (%ld; expected %ld)\n", + spec_to_string(device).c_str(), video_frame.len - video_offset, expected_length); + } + if (video_frame.owner) { + video_frame.owner->release_frame(video_frame); + } + + // Still send on the information that we _had_ a frame, even though it's corrupted, + // so that pts can go up accordingly. + { + unique_lock lock(card_mutex); + CaptureCard::NewFrame new_frame; + new_frame.frame = RefCountedFrame(FrameAllocator::Frame()); + new_frame.length = frame_length; + new_frame.interlaced = false; + new_frame.dropped_frames = dropped_frames; + new_frame.received_timestamp = video_frame.received_timestamp; + card->new_frames.push_back(move(new_frame)); + card->jitter_history.frame_arrived(video_frame.received_timestamp, frame_length, dropped_frames); + } + card->new_frames_changed.notify_all(); + return; + } + + unsigned num_fields = video_format.interlaced ? 2 : 1; + steady_clock::time_point frame_upload_start; + bool interlaced_stride = false; + if (video_format.interlaced) { + // Send the two fields along as separate frames; the other side will need to add + // a deinterlacer to actually get this right. + assert(video_format.height % 2 == 0); + video_format.height /= 2; + cbcr_height /= 2; + assert(frame_length % 2 == 0); + frame_length /= 2; + num_fields = 2; + if (video_format.second_field_start == 1) { + interlaced_stride = true; + } + frame_upload_start = steady_clock::now(); + } + userdata->last_interlaced = video_format.interlaced; + userdata->last_has_signal = video_format.has_signal; + userdata->last_is_connected = video_format.is_connected; + userdata->last_frame_rate_nom = video_format.frame_rate_nom; + userdata->last_frame_rate_den = video_format.frame_rate_den; + RefCountedFrame frame(video_frame); + + // Upload the textures. + for (unsigned field = 0; field < num_fields; ++field) { + // Put the actual texture upload in a lambda that is executed in the main thread. + // It is entirely possible to do this in the same thread (and it might even be + // faster, depending on the GPU and driver), but it appears to be trickling + // driver bugs very easily. + // + // Note that this means we must hold on to the actual frame data in + // until the upload command is run, but we hold on to much longer than that + // (in fact, all the way until we no longer use the texture in rendering). + auto upload_func = [this, field, video_format, y_offset, video_offset, cbcr_offset, cbcr_width, cbcr_height, interlaced_stride, userdata]() { + unsigned field_start_line; + if (field == 1) { + field_start_line = video_format.second_field_start; + } else { + field_start_line = video_format.extra_lines_top; + } + + // For anything not FRAME_FORMAT_YCBCR_10BIT, v210_width will be nonsensical but not used. + size_t v210_width = video_format.stride / sizeof(uint32_t); + ensure_texture_resolution(userdata, field, video_format.width, video_format.height, cbcr_width, cbcr_height, v210_width); + + glBindBuffer(GL_PIXEL_UNPACK_BUFFER, userdata->pbo); + check_error(); + + switch (userdata->pixel_format) { + case PixelFormat_10BitYCbCr: { + size_t field_start = video_offset + video_format.stride * field_start_line; + upload_texture(userdata->tex_v210[field], v210_width, video_format.height, video_format.stride, interlaced_stride, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, field_start); + v210_converter->convert(userdata->tex_v210[field], userdata->tex_444[field], video_format.width, video_format.height); + break; + } + case PixelFormat_8BitYCbCr: { + size_t field_y_start = y_offset + video_format.width * field_start_line; + size_t field_cbcr_start = cbcr_offset + cbcr_width * field_start_line * sizeof(uint16_t); + + // Make up our own strides, since we are interleaving. + upload_texture(userdata->tex_y[field], video_format.width, video_format.height, video_format.width, interlaced_stride, GL_RED, GL_UNSIGNED_BYTE, field_y_start); + upload_texture(userdata->tex_cbcr[field], cbcr_width, cbcr_height, cbcr_width * sizeof(uint16_t), interlaced_stride, GL_RG, GL_UNSIGNED_BYTE, field_cbcr_start); + break; + } + case PixelFormat_8BitYCbCrPlanar: { + assert(field_start_line == 0); // We don't really support interlaced here. + size_t field_y_start = y_offset; + size_t field_cb_start = cbcr_offset; + size_t field_cr_start = cbcr_offset + cbcr_width * cbcr_height; + + // Make up our own strides, since we are interleaving. + upload_texture(userdata->tex_y[field], video_format.width, video_format.height, video_format.width, interlaced_stride, GL_RED, GL_UNSIGNED_BYTE, field_y_start); + upload_texture(userdata->tex_cb[field], cbcr_width, cbcr_height, cbcr_width, interlaced_stride, GL_RED, GL_UNSIGNED_BYTE, field_cb_start); + upload_texture(userdata->tex_cr[field], cbcr_width, cbcr_height, cbcr_width, interlaced_stride, GL_RED, GL_UNSIGNED_BYTE, field_cr_start); + break; + } + case PixelFormat_8BitBGRA: { + size_t field_start = video_offset + video_format.stride * field_start_line; + upload_texture(userdata->tex_rgba[field], video_format.width, video_format.height, video_format.stride, interlaced_stride, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, field_start); + // These could be asked to deliver mipmaps at any time. + glBindTexture(GL_TEXTURE_2D, userdata->tex_rgba[field]); + check_error(); + glGenerateMipmap(GL_TEXTURE_2D); + check_error(); + glBindTexture(GL_TEXTURE_2D, 0); + check_error(); + break; + } + default: + assert(false); + } + + glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); + check_error(); + }; + + if (field == 1) { + // Don't upload the second field as fast as we can; wait until + // the field time has approximately passed. (Otherwise, we could + // get timing jitter against the other sources, and possibly also + // against the video display, although the latter is not as critical.) + // This requires our system clock to be reasonably close to the + // video clock, but that's not an unreasonable assumption. + steady_clock::time_point second_field_start = frame_upload_start + + nanoseconds(frame_length * 1000000000 / TIMEBASE); + this_thread::sleep_until(second_field_start); + } + + { + unique_lock lock(card_mutex); + CaptureCard::NewFrame new_frame; + new_frame.frame = frame; + new_frame.length = frame_length; + new_frame.field = field; + new_frame.interlaced = video_format.interlaced; + new_frame.upload_func = upload_func; + new_frame.dropped_frames = dropped_frames; + new_frame.received_timestamp = video_frame.received_timestamp; // Ignore the audio timestamp. + card->new_frames.push_back(move(new_frame)); + card->jitter_history.frame_arrived(video_frame.received_timestamp, frame_length, dropped_frames); + card->may_have_dropped_last_frame = false; + } + card->new_frames_changed.notify_all(); + } +} + +void Mixer::bm_hotplug_add(libusb_device *dev) +{ + lock_guard lock(hotplug_mutex); + hotplugged_cards.push_back(dev); +} + +void Mixer::bm_hotplug_remove(unsigned card_index) +{ + cards[card_index].new_frames_changed.notify_all(); +} + +void Mixer::thread_func() +{ + pthread_setname_np(pthread_self(), "Mixer_OpenGL"); + + eglBindAPI(EGL_OPENGL_API); + QOpenGLContext *context = create_context(mixer_surface); + if (!make_current(context, mixer_surface)) { + printf("oops\n"); + exit(1); + } + + // Start the actual capture. (We don't want to do it before we're actually ready + // to process output frames.) + for (unsigned card_index = 0; card_index < num_cards + num_video_inputs + num_html_inputs; ++card_index) { + if (int(card_index) != output_card_index) { + cards[card_index].capture->start_bm_capture(); + } + } + + BasicStats basic_stats(/*verbose=*/true, /*use_opengl=*/true); + int stats_dropped_frames = 0; + + while (!should_quit) { + if (desired_output_card_index != output_card_index) { + set_output_card_internal(desired_output_card_index); + } + if (output_card_index != -1 && + desired_output_video_mode != output_video_mode) { + DeckLinkOutput *output = cards[output_card_index].output.get(); + output->end_output(); + desired_output_video_mode = output_video_mode = output->pick_video_mode(desired_output_video_mode); + output->start_output(desired_output_video_mode, pts_int); + } + + CaptureCard::NewFrame new_frames[MAX_VIDEO_CARDS]; + bool has_new_frame[MAX_VIDEO_CARDS] = { false }; + + bool master_card_is_output; + unsigned master_card_index; + if (output_card_index != -1) { + master_card_is_output = true; + master_card_index = output_card_index; + } else { + master_card_is_output = false; + master_card_index = theme->map_signal(master_clock_channel); + assert(master_card_index < num_cards + num_video_inputs); + } + + OutputFrameInfo output_frame_info = get_one_frame_from_each_card(master_card_index, master_card_is_output, new_frames, has_new_frame); + schedule_audio_resampling_tasks(output_frame_info.dropped_frames, output_frame_info.num_samples, output_frame_info.frame_duration, output_frame_info.is_preroll, output_frame_info.frame_timestamp); + stats_dropped_frames += output_frame_info.dropped_frames; + + handle_hotplugged_cards(); + + for (unsigned card_index = 0; card_index < num_cards + num_video_inputs + num_html_inputs; ++card_index) { + DeviceSpec device = card_index_to_device(card_index, num_cards); + if (card_index == master_card_index || !has_new_frame[card_index]) { + continue; + } + if (new_frames[card_index].frame->len == 0) { + ++new_frames[card_index].dropped_frames; + } + if (new_frames[card_index].dropped_frames > 0) { + printf("%s dropped %d frames before this\n", + spec_to_string(device).c_str(), int(new_frames[card_index].dropped_frames)); + } + } + + // If the first card is reporting a corrupted or otherwise dropped frame, + // just increase the pts (skipping over this frame) and don't try to compute anything new. + if (!master_card_is_output && new_frames[master_card_index].frame->len == 0) { + ++stats_dropped_frames; + pts_int += new_frames[master_card_index].length; + continue; + } + + for (unsigned card_index = 0; card_index < num_cards + num_video_inputs + num_html_inputs; ++card_index) { + if (!has_new_frame[card_index] || new_frames[card_index].frame->len == 0) + continue; + + CaptureCard::NewFrame *new_frame = &new_frames[card_index]; + assert(new_frame->frame != nullptr); + insert_new_frame(new_frame->frame, new_frame->field, new_frame->interlaced, card_index, &input_state); + check_error(); + + // The new texture might need uploading before use. + if (new_frame->upload_func) { + new_frame->upload_func(); + new_frame->upload_func = nullptr; + } + } + + int64_t frame_duration = output_frame_info.frame_duration; + render_one_frame(frame_duration); + { + lock_guard lock(frame_num_mutex); + ++frame_num; + } + frame_num_updated.notify_all(); + pts_int += frame_duration; + + basic_stats.update(frame_num, stats_dropped_frames); + // if (frame_num % 100 == 0) chain->print_phase_timing(); + + if (should_cut.exchange(false)) { // Test and clear. + video_encoder->do_cut(frame_num); + } + +#if 0 + // Reset every 100 frames, so that local variations in frame times + // (especially for the first few frames, when the shaders are + // compiled etc.) don't make it hard to measure for the entire + // remaining duration of the program. + if (frame == 10000) { + frame = 0; + start = now; + } +#endif + check_error(); + } + + resource_pool->clean_context(); +} + +bool Mixer::input_card_is_master_clock(unsigned card_index, unsigned master_card_index) const +{ + if (output_card_index != -1) { + // The output card (ie., cards[output_card_index].output) is the master clock, + // so no input card (ie., cards[card_index].capture) is. + return false; + } + return (card_index == master_card_index); +} + +void Mixer::trim_queue(CaptureCard *card, size_t safe_queue_length) +{ + // Count the number of frames in the queue, including any frames + // we dropped. It's hard to know exactly how we should deal with + // dropped (corrupted) input frames; they don't help our goal of + // avoiding starvation, but they still add to the problem of latency. + // Since dropped frames is going to mean a bump in the signal anyway, + // we err on the side of having more stable latency instead. + unsigned queue_length = 0; + for (const CaptureCard::NewFrame &frame : card->new_frames) { + queue_length += frame.dropped_frames + 1; + } + + // If needed, drop frames until the queue is below the safe limit. + // We prefer to drop from the head, because all else being equal, + // we'd like more recent frames (less latency). + unsigned dropped_frames = 0; + while (queue_length > safe_queue_length) { + assert(!card->new_frames.empty()); + assert(queue_length > card->new_frames.front().dropped_frames); + queue_length -= card->new_frames.front().dropped_frames; + + if (queue_length <= safe_queue_length) { + // No need to drop anything. + break; + } + + card->new_frames.pop_front(); + card->new_frames_changed.notify_all(); + --queue_length; + ++dropped_frames; + + if (queue_length == 0 && card->is_cef_capture) { + card->may_have_dropped_last_frame = true; + } + } + + card->metric_input_dropped_frames_jitter += dropped_frames; + card->metric_input_queue_length_frames = queue_length; + +#if 0 + if (dropped_frames > 0) { + fprintf(stderr, "Card %u dropped %u frame(s) to keep latency down.\n", + card_index, dropped_frames); + } +#endif +} + +pair Mixer::get_channels_json() +{ + Channels ret; + for (int channel_idx = 2; channel_idx < theme->get_num_channels(); ++channel_idx) { + Channel *channel = ret.add_channel(); + channel->set_index(channel_idx); + channel->set_name(theme->get_channel_name(channel_idx)); + channel->set_color(theme->get_channel_color(channel_idx)); + } + string contents; + google::protobuf::util::MessageToJsonString(ret, &contents); // Ignore any errors. + return make_pair(contents, "text/json"); +} + +pair Mixer::get_channel_color_http(unsigned channel_idx) +{ + return make_pair(theme->get_channel_color(channel_idx), "text/plain"); +} + +Mixer::OutputFrameInfo Mixer::get_one_frame_from_each_card(unsigned master_card_index, bool master_card_is_output, CaptureCard::NewFrame new_frames[MAX_VIDEO_CARDS], bool has_new_frame[MAX_VIDEO_CARDS]) +{ + OutputFrameInfo output_frame_info; +start: + unique_lock lock(card_mutex, defer_lock); + if (master_card_is_output) { + // Clocked to the output, so wait for it to be ready for the next frame. + cards[master_card_index].output->wait_for_frame(pts_int, &output_frame_info.dropped_frames, &output_frame_info.frame_duration, &output_frame_info.is_preroll, &output_frame_info.frame_timestamp); + lock.lock(); + } else { + // Wait for the master card to have a new frame. + // TODO: Add a timeout. + output_frame_info.is_preroll = false; + lock.lock(); + cards[master_card_index].new_frames_changed.wait(lock, [this, master_card_index]{ return !cards[master_card_index].new_frames.empty() || cards[master_card_index].capture->get_disconnected(); }); + } + + if (master_card_is_output) { + handle_hotplugged_cards(); + } else if (cards[master_card_index].new_frames.empty()) { + // We were woken up, but not due to a new frame. Deal with it + // and then restart. + assert(cards[master_card_index].capture->get_disconnected()); + handle_hotplugged_cards(); + lock.unlock(); + goto start; + } + + for (unsigned card_index = 0; card_index < num_cards + num_video_inputs + num_html_inputs; ++card_index) { + CaptureCard *card = &cards[card_index]; + if (card->new_frames.empty()) { // Starvation. + ++card->metric_input_duped_frames; +#ifdef HAVE_CEF + if (card->is_cef_capture && card->may_have_dropped_last_frame) { + // Unlike other sources, CEF is not guaranteed to send us a steady + // stream of frames, so we'll have to ask it to repaint the frame + // we dropped. (may_have_dropped_last_frame is set whenever we + // trim the queue completely away, and cleared when we actually + // get a new frame.) + ((CEFCapture *)card->capture.get())->request_new_frame(); + } +#endif + } else { + new_frames[card_index] = move(card->new_frames.front()); + has_new_frame[card_index] = true; + card->new_frames.pop_front(); + card->new_frames_changed.notify_all(); + } + } + + if (!master_card_is_output) { + output_frame_info.frame_timestamp = new_frames[master_card_index].received_timestamp; + output_frame_info.dropped_frames = new_frames[master_card_index].dropped_frames; + output_frame_info.frame_duration = new_frames[master_card_index].length; + } + + if (!output_frame_info.is_preroll) { + output_jitter_history.frame_arrived(output_frame_info.frame_timestamp, output_frame_info.frame_duration, output_frame_info.dropped_frames); + } + + for (unsigned card_index = 0; card_index < num_cards + num_video_inputs + num_html_inputs; ++card_index) { + CaptureCard *card = &cards[card_index]; + if (has_new_frame[card_index] && + !input_card_is_master_clock(card_index, master_card_index) && + !output_frame_info.is_preroll) { + card->queue_length_policy.update_policy( + output_frame_info.frame_timestamp, + card->jitter_history.get_expected_next_frame(), + new_frames[master_card_index].length, + output_frame_info.frame_duration, + card->jitter_history.estimate_max_jitter(), + output_jitter_history.estimate_max_jitter()); + trim_queue(card, min(global_flags.max_input_queue_frames, + card->queue_length_policy.get_safe_queue_length())); + } + } + + // This might get off by a fractional sample when changing master card + // between ones with different frame rates, but that's fine. + int num_samples_times_timebase = OUTPUT_FREQUENCY * output_frame_info.frame_duration + fractional_samples; + output_frame_info.num_samples = num_samples_times_timebase / TIMEBASE; + fractional_samples = num_samples_times_timebase % TIMEBASE; + assert(output_frame_info.num_samples >= 0); + + return output_frame_info; +} + +void Mixer::handle_hotplugged_cards() +{ + // Check for cards that have been disconnected since last frame. + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + CaptureCard *card = &cards[card_index]; + if (card->capture->get_disconnected()) { + fprintf(stderr, "Card %u went away, replacing with a fake card.\n", card_index); + FakeCapture *capture = new FakeCapture(global_flags.width, global_flags.height, FAKE_FPS, OUTPUT_FREQUENCY, card_index, global_flags.fake_cards_audio); + configure_card(card_index, capture, CardType::FAKE_CAPTURE, /*output=*/nullptr); + card->queue_length_policy.reset(card_index); + card->capture->start_bm_capture(); + } + } + + // Check for cards that have been connected since last frame. + vector hotplugged_cards_copy; + { + lock_guard lock(hotplug_mutex); + swap(hotplugged_cards, hotplugged_cards_copy); + } + for (libusb_device *new_dev : hotplugged_cards_copy) { + // Look for a fake capture card where we can stick this in. + int free_card_index = -1; + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + if (cards[card_index].is_fake_capture) { + free_card_index = card_index; + break; + } + } + + if (free_card_index == -1) { + fprintf(stderr, "New card plugged in, but no free slots -- ignoring.\n"); + libusb_unref_device(new_dev); + } else { + // BMUSBCapture takes ownership. + fprintf(stderr, "New card plugged in, choosing slot %d.\n", free_card_index); + CaptureCard *card = &cards[free_card_index]; + BMUSBCapture *capture = new BMUSBCapture(free_card_index, new_dev); + configure_card(free_card_index, capture, CardType::LIVE_CARD, /*output=*/nullptr); + card->queue_length_policy.reset(free_card_index); + capture->set_card_disconnected_callback(bind(&Mixer::bm_hotplug_remove, this, free_card_index)); + capture->start_bm_capture(); + } + } +} + + +void Mixer::schedule_audio_resampling_tasks(unsigned dropped_frames, int num_samples_per_frame, int length_per_frame, bool is_preroll, steady_clock::time_point frame_timestamp) +{ + // Resample the audio as needed, including from previously dropped frames. + assert(num_cards > 0); + for (unsigned frame_num = 0; frame_num < dropped_frames + 1; ++frame_num) { + const bool dropped_frame = (frame_num != dropped_frames); + { + // Signal to the audio thread to process this frame. + // Note that if the frame is a dropped frame, we signal that + // we don't want to use this frame as base for adjusting + // the resampler rate. The reason for this is that the timing + // of these frames is often way too late; they typically don't + // “arrive” before we synthesize them. Thus, we could end up + // in a situation where we have inserted e.g. five audio frames + // into the queue before we then start pulling five of them + // back out. This makes ResamplingQueue overestimate the delay, + // causing undue resampler changes. (We _do_ use the last, + // non-dropped frame; perhaps we should just discard that as well, + // since dropped frames are expected to be rare, and it might be + // better to just wait until we have a slightly more normal situation). + unique_lock lock(audio_mutex); + bool adjust_rate = !dropped_frame && !is_preroll; + audio_task_queue.push(AudioTask{pts_int, num_samples_per_frame, adjust_rate, frame_timestamp}); + audio_task_queue_changed.notify_one(); + } + if (dropped_frame) { + // For dropped frames, increase the pts. Note that if the format changed + // in the meantime, we have no way of detecting that; we just have to + // assume the frame length is always the same. + pts_int += length_per_frame; + } + } +} + +void Mixer::render_one_frame(int64_t duration) +{ + // Determine the time code for this frame before we start rendering. + string timecode_text = timecode_renderer->get_timecode_text(double(pts_int) / TIMEBASE, frame_num); + if (display_timecode_on_stdout) { + printf("Timecode: '%s'\n", timecode_text.c_str()); + } + + // Update Y'CbCr settings for all cards. + { + unique_lock lock(card_mutex); + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + YCbCrInterpretation *interpretation = &ycbcr_interpretation[card_index]; + input_state.ycbcr_coefficients_auto[card_index] = interpretation->ycbcr_coefficients_auto; + input_state.ycbcr_coefficients[card_index] = interpretation->ycbcr_coefficients; + input_state.full_range[card_index] = interpretation->full_range; + } + } + + // Get the main chain from the theme, and set its state immediately. + Theme::Chain theme_main_chain = theme->get_chain(0, pts(), global_flags.width, global_flags.height, input_state); + EffectChain *chain = theme_main_chain.chain; + theme_main_chain.setup_chain(); + //theme_main_chain.chain->enable_phase_timing(true); + + // If HDMI/SDI output is active and the user has requested auto mode, + // its mode overrides the existing Y'CbCr setting for the chain. + YCbCrLumaCoefficients ycbcr_output_coefficients; + if (global_flags.ycbcr_auto_coefficients && output_card_index != -1) { + ycbcr_output_coefficients = cards[output_card_index].output->preferred_ycbcr_coefficients(); + } else { + ycbcr_output_coefficients = global_flags.ycbcr_rec709_coefficients ? YCBCR_REC_709 : YCBCR_REC_601; + } + + // TODO: Reduce the duplication against theme.cpp. + YCbCrFormat output_ycbcr_format; + output_ycbcr_format.chroma_subsampling_x = 1; + output_ycbcr_format.chroma_subsampling_y = 1; + output_ycbcr_format.luma_coefficients = ycbcr_output_coefficients; + output_ycbcr_format.full_range = false; + output_ycbcr_format.num_levels = 1 << global_flags.x264_bit_depth; + chain->change_ycbcr_output_format(output_ycbcr_format); + + // Render main chain. If we're using zerocopy Quick Sync encoding + // (the default case), we take an extra copy of the created outputs, + // so that we can display it back to the screen later (it's less memory + // bandwidth than writing and reading back an RGBA texture, even at 16-bit). + // Ideally, we'd like to avoid taking copies and just use the main textures + // for display as well, but they're just views into VA-API memory and must be + // unmapped during encoding, so we can't use them for display, unfortunately. + GLuint y_tex, cbcr_full_tex, cbcr_tex; + GLuint y_copy_tex, cbcr_copy_tex = 0; + GLuint y_display_tex, cbcr_display_tex; + GLenum y_type = (global_flags.x264_bit_depth > 8) ? GL_R16 : GL_R8; + GLenum cbcr_type = (global_flags.x264_bit_depth > 8) ? GL_RG16 : GL_RG8; + const bool is_zerocopy = video_encoder->is_zerocopy(); + if (is_zerocopy) { + cbcr_full_tex = resource_pool->create_2d_texture(cbcr_type, global_flags.width, global_flags.height); + y_copy_tex = resource_pool->create_2d_texture(y_type, global_flags.width, global_flags.height); + cbcr_copy_tex = resource_pool->create_2d_texture(cbcr_type, global_flags.width / 2, global_flags.height / 2); + + y_display_tex = y_copy_tex; + cbcr_display_tex = cbcr_copy_tex; + + // y_tex and cbcr_tex will be given by VideoEncoder. + } else { + cbcr_full_tex = resource_pool->create_2d_texture(cbcr_type, global_flags.width, global_flags.height); + y_tex = resource_pool->create_2d_texture(y_type, global_flags.width, global_flags.height); + cbcr_tex = resource_pool->create_2d_texture(cbcr_type, global_flags.width / 2, global_flags.height / 2); + + y_display_tex = y_tex; + cbcr_display_tex = cbcr_tex; + } + + const int64_t av_delay = lrint(global_flags.audio_queue_length_ms * 0.001 * TIMEBASE); // Corresponds to the delay in ResamplingQueue. + bool got_frame = video_encoder->begin_frame(pts_int + av_delay, duration, ycbcr_output_coefficients, theme_main_chain.input_frames, &y_tex, &cbcr_tex); + assert(got_frame); + + GLuint fbo; + if (is_zerocopy) { + fbo = resource_pool->create_fbo(y_tex, cbcr_full_tex, y_copy_tex); + } else { + fbo = resource_pool->create_fbo(y_tex, cbcr_full_tex); + } + check_error(); + chain->render_to_fbo(fbo, global_flags.width, global_flags.height); + + if (display_timecode_in_stream) { + // Render the timecode on top. + timecode_renderer->render_timecode(fbo, timecode_text); + } + + resource_pool->release_fbo(fbo); + + if (is_zerocopy) { + chroma_subsampler->subsample_chroma(cbcr_full_tex, global_flags.width, global_flags.height, cbcr_tex, cbcr_copy_tex); + } else { + chroma_subsampler->subsample_chroma(cbcr_full_tex, global_flags.width, global_flags.height, cbcr_tex); + } + if (output_card_index != -1) { + cards[output_card_index].output->send_frame(y_tex, cbcr_full_tex, ycbcr_output_coefficients, theme_main_chain.input_frames, pts_int, duration); + } + resource_pool->release_2d_texture(cbcr_full_tex); + + // Set the right state for the Y' and CbCr textures we use for display. + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glBindTexture(GL_TEXTURE_2D, y_display_tex); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + glBindTexture(GL_TEXTURE_2D, cbcr_display_tex); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + RefCountedGLsync fence = video_encoder->end_frame(); + + // The live frame pieces the Y'CbCr texture copies back into RGB and displays them. + // It owns y_display_tex and cbcr_display_tex now (whichever textures they are). + DisplayFrame live_frame; + live_frame.chain = display_chain.get(); + live_frame.setup_chain = [this, y_display_tex, cbcr_display_tex]{ + display_input->set_texture_num(0, y_display_tex); + display_input->set_texture_num(1, cbcr_display_tex); + }; + live_frame.ready_fence = fence; + live_frame.input_frames = {}; + live_frame.temp_textures = { y_display_tex, cbcr_display_tex }; + output_channel[OUTPUT_LIVE].output_frame(move(live_frame)); + + // Set up preview and any additional channels. + for (int i = 1; i < theme->get_num_channels() + 2; ++i) { + DisplayFrame display_frame; + Theme::Chain chain = theme->get_chain(i, pts(), global_flags.width, global_flags.height, input_state); // FIXME: dimensions + display_frame.chain = move(chain.chain); + display_frame.setup_chain = move(chain.setup_chain); + display_frame.ready_fence = fence; + display_frame.input_frames = move(chain.input_frames); + display_frame.temp_textures = {}; + output_channel[i].output_frame(move(display_frame)); + } +} + +void Mixer::audio_thread_func() +{ + pthread_setname_np(pthread_self(), "Mixer_Audio"); + + while (!should_quit) { + AudioTask task; + + { + unique_lock lock(audio_mutex); + audio_task_queue_changed.wait(lock, [this]{ return should_quit || !audio_task_queue.empty(); }); + if (should_quit) { + return; + } + task = audio_task_queue.front(); + audio_task_queue.pop(); + } + + ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy = + task.adjust_rate ? ResamplingQueue::ADJUST_RATE : ResamplingQueue::DO_NOT_ADJUST_RATE; + vector samples_out = audio_mixer->get_output( + task.frame_timestamp, + task.num_samples, + rate_adjustment_policy); + + // Send the samples to the sound card, then add them to the output. + if (alsa) { + alsa->write(samples_out); + } + if (output_card_index != -1) { + const int64_t av_delay = lrint(global_flags.audio_queue_length_ms * 0.001 * TIMEBASE); // Corresponds to the delay in ResamplingQueue. + cards[output_card_index].output->send_audio(task.pts_int + av_delay, samples_out); + } + video_encoder->add_audio(task.pts_int, move(samples_out)); + } +} + +void Mixer::release_display_frame(DisplayFrame *frame) +{ + for (GLuint texnum : frame->temp_textures) { + resource_pool->release_2d_texture(texnum); + } + frame->temp_textures.clear(); + frame->ready_fence.reset(); + frame->input_frames.clear(); +} + +void Mixer::start() +{ + mixer_thread = thread(&Mixer::thread_func, this); + audio_thread = thread(&Mixer::audio_thread_func, this); +} + +void Mixer::quit() +{ + should_quit = true; + audio_task_queue_changed.notify_one(); + mixer_thread.join(); + audio_thread.join(); +} + +void Mixer::transition_clicked(int transition_num) +{ + theme->transition_clicked(transition_num, pts()); +} + +void Mixer::channel_clicked(int preview_num) +{ + theme->channel_clicked(preview_num); +} + +YCbCrInterpretation Mixer::get_input_ycbcr_interpretation(unsigned card_index) const +{ + unique_lock lock(card_mutex); + return ycbcr_interpretation[card_index]; +} + +void Mixer::set_input_ycbcr_interpretation(unsigned card_index, const YCbCrInterpretation &interpretation) +{ + unique_lock lock(card_mutex); + ycbcr_interpretation[card_index] = interpretation; +} + +void Mixer::start_mode_scanning(unsigned card_index) +{ + assert(card_index < num_cards); + if (is_mode_scanning[card_index]) { + return; + } + is_mode_scanning[card_index] = true; + mode_scanlist[card_index].clear(); + for (const auto &mode : cards[card_index].capture->get_available_video_modes()) { + mode_scanlist[card_index].push_back(mode.first); + } + assert(!mode_scanlist[card_index].empty()); + mode_scanlist_index[card_index] = 0; + cards[card_index].capture->set_video_mode(mode_scanlist[card_index][0]); + last_mode_scan_change[card_index] = steady_clock::now(); +} + +map Mixer::get_available_output_video_modes() const +{ + assert(desired_output_card_index != -1); + unique_lock lock(card_mutex); + return cards[desired_output_card_index].output->get_available_video_modes(); +} + +string Mixer::get_ffmpeg_filename(unsigned card_index) const +{ + assert(card_index >= num_cards && card_index < num_cards + num_video_inputs); + return ((FFmpegCapture *)(cards[card_index].capture.get()))->get_filename(); +} + +void Mixer::set_ffmpeg_filename(unsigned card_index, const string &filename) { + assert(card_index >= num_cards && card_index < num_cards + num_video_inputs); + ((FFmpegCapture *)(cards[card_index].capture.get()))->change_filename(filename); +} + +void Mixer::wait_for_next_frame() +{ + unique_lock lock(frame_num_mutex); + unsigned old_frame_num = frame_num; + frame_num_updated.wait_for(lock, seconds(1), // Timeout is just in case. + [old_frame_num, this]{ return this->frame_num > old_frame_num; }); +} + +Mixer::OutputChannel::~OutputChannel() +{ + if (has_current_frame) { + parent->release_display_frame(¤t_frame); + } + if (has_ready_frame) { + parent->release_display_frame(&ready_frame); + } +} + +void Mixer::OutputChannel::output_frame(DisplayFrame &&frame) +{ + // Store this frame for display. Remove the ready frame if any + // (it was seemingly never used). + { + unique_lock lock(frame_mutex); + if (has_ready_frame) { + parent->release_display_frame(&ready_frame); + } + ready_frame = move(frame); + has_ready_frame = true; + + // Call the callbacks under the mutex (they should be short), + // so that we don't race against a callback removal. + for (const auto &key_and_callback : new_frame_ready_callbacks) { + key_and_callback.second(); + } + } + + // Reduce the number of callbacks by filtering duplicates. The reason + // why we bother doing this is that Qt seemingly can get into a state + // where its builds up an essentially unbounded queue of signals, + // consuming more and more memory, and there's no good way of collapsing + // user-defined signals or limiting the length of the queue. + if (transition_names_updated_callback) { + vector transition_names = global_mixer->get_transition_names(); + bool changed = false; + if (transition_names.size() != last_transition_names.size()) { + changed = true; + } else { + for (unsigned i = 0; i < transition_names.size(); ++i) { + if (transition_names[i] != last_transition_names[i]) { + changed = true; + break; + } + } + } + if (changed) { + transition_names_updated_callback(transition_names); + last_transition_names = transition_names; + } + } + if (name_updated_callback) { + string name = global_mixer->get_channel_name(channel); + if (name != last_name) { + name_updated_callback(name); + last_name = name; + } + } + if (color_updated_callback) { + string color = global_mixer->get_channel_color(channel); + if (color != last_color) { + color_updated_callback(color); + last_color = color; + } + } +} + +bool Mixer::OutputChannel::get_display_frame(DisplayFrame *frame) +{ + unique_lock lock(frame_mutex); + if (!has_current_frame && !has_ready_frame) { + return false; + } + + if (has_current_frame && has_ready_frame) { + // We have a new ready frame. Toss the current one. + parent->release_display_frame(¤t_frame); + has_current_frame = false; + } + if (has_ready_frame) { + assert(!has_current_frame); + current_frame = move(ready_frame); + ready_frame.ready_fence.reset(); // Drop the refcount. + ready_frame.input_frames.clear(); // Drop the refcounts. + has_current_frame = true; + has_ready_frame = false; + } + + *frame = current_frame; + return true; +} + +void Mixer::OutputChannel::add_frame_ready_callback(void *key, Mixer::new_frame_ready_callback_t callback) +{ + unique_lock lock(frame_mutex); + new_frame_ready_callbacks[key] = callback; +} + +void Mixer::OutputChannel::remove_frame_ready_callback(void *key) +{ + unique_lock lock(frame_mutex); + new_frame_ready_callbacks.erase(key); +} + +void Mixer::OutputChannel::set_transition_names_updated_callback(Mixer::transition_names_updated_callback_t callback) +{ + transition_names_updated_callback = callback; +} + +void Mixer::OutputChannel::set_name_updated_callback(Mixer::name_updated_callback_t callback) +{ + name_updated_callback = callback; +} + +void Mixer::OutputChannel::set_color_updated_callback(Mixer::color_updated_callback_t callback) +{ + color_updated_callback = callback; +} + +mutex RefCountedGLsync::fence_lock; diff --git a/nageru/mixer.h b/nageru/mixer.h new file mode 100644 index 0000000..32a1ea4 --- /dev/null +++ b/nageru/mixer.h @@ -0,0 +1,637 @@ +#ifndef _MIXER_H +#define _MIXER_H 1 + +// The actual video mixer, running in its own separate background thread. + +#include +#include + +#undef Success + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "audio_mixer.h" +#include "bmusb/bmusb.h" +#include "defs.h" +#include "httpd.h" +#include "input_state.h" +#include "libusb.h" +#include "pbo_frame_allocator.h" +#include "ref_counted_frame.h" +#include "ref_counted_gl_sync.h" +#include "theme.h" +#include "timebase.h" +#include "video_encoder.h" +#include "ycbcr_interpretation.h" + +class ALSAOutput; +class ChromaSubsampler; +class DeckLinkOutput; +class QSurface; +class QSurfaceFormat; +class TimecodeRenderer; +class v210Converter; + +namespace movit { +class Effect; +class EffectChain; +class ResourcePool; +class YCbCrInput; +} // namespace movit + +// A class to estimate the future jitter. Used in QueueLengthPolicy (see below). +// +// There are many ways to estimate jitter; I've tested a few ones (and also +// some algorithms that don't explicitly model jitter) with different +// parameters on some real-life data in experiments/queue_drop_policy.cpp. +// This is one based on simple order statistics where I've added some margin in +// the number of starvation events; I believe that about one every hour would +// probably be acceptable, but this one typically goes lower than that, at the +// cost of 2–3 ms extra latency. (If the queue is hard-limited to one frame, it's +// possible to get ~10 ms further down, but this would mean framedrops every +// second or so.) The general strategy is: Take the 99.9-percentile jitter over +// last 5000 frames, multiply by two, and that's our worst-case jitter +// estimate. The fact that we're not using the max value means that we could +// actually even throw away very late frames immediately, which means we only +// get one user-visible event instead of seeing something both when the frame +// arrives late (duplicate frame) and then again when we drop. +class JitterHistory { +private: + static constexpr size_t history_length = 5000; + static constexpr double percentile = 0.999; + static constexpr double multiplier = 2.0; + +public: + void register_metrics(const std::vector> &labels); + void unregister_metrics(const std::vector> &labels); + + void clear() { + history.clear(); + orders.clear(); + } + void frame_arrived(std::chrono::steady_clock::time_point now, int64_t frame_duration, size_t dropped_frames); + std::chrono::steady_clock::time_point get_expected_next_frame() const { return expected_timestamp; } + double estimate_max_jitter() const; + +private: + // A simple O(k) based algorithm for getting the k-th largest or + // smallest element from our window; we simply keep the multiset + // ordered (insertions and deletions are O(n) as always) and then + // iterate from one of the sides. If we had larger values of k, + // we could go for a more complicated setup with two sets or heaps + // (one increasing and one decreasing) that we keep balanced around + // the point, or it is possible to reimplement std::set with + // counts in each node. However, since k=5, we don't need this. + std::multiset orders; + std::deque::iterator> history; + + std::chrono::steady_clock::time_point expected_timestamp = std::chrono::steady_clock::time_point::min(); + + // Metrics. There are no direct summaries for jitter, since we already have latency summaries. + std::atomic metric_input_underestimated_jitter_frames{0}; + std::atomic metric_input_estimated_max_jitter_seconds{0.0 / 0.0}; +}; + +// For any card that's not the master (where we pick out the frames as they +// come, as fast as we can process), there's going to be a queue. The question +// is when we should drop frames from that queue (apart from the obvious +// dropping if the 16-frame queue should become full), especially given that +// the frame rate could be lower or higher than the master (either subtly or +// dramatically). We have two (conflicting) demands: +// +// 1. We want to avoid starving the queue. +// 2. We don't want to add more delay than is needed. +// +// Our general strategy is to drop as many frames as we can (helping for #2) +// that we think is safe for #1 given jitter. To this end, we measure the +// deviation from the expected arrival time for all cards, and use that for +// continuous jitter estimation. +// +// We then drop everything from the queue that we're sure we won't need to +// serve the output in the time before the next frame arrives. Typically, +// this means the queue will contain 0 or 1 frames, although more is also +// possible if the jitter is very high. +class QueueLengthPolicy { +public: + QueueLengthPolicy() {} + void reset(unsigned card_index) { + this->card_index = card_index; + } + + void register_metrics(const std::vector> &labels); + void unregister_metrics(const std::vector> &labels); + + // Call after picking out a frame, so 0 means starvation. + void update_policy(std::chrono::steady_clock::time_point now, + std::chrono::steady_clock::time_point expected_next_frame, + int64_t input_frame_duration, + int64_t master_frame_duration, + double max_input_card_jitter_seconds, + double max_master_card_jitter_seconds); + unsigned get_safe_queue_length() const { return safe_queue_length; } + +private: + unsigned card_index; // For debugging and metrics only. + unsigned safe_queue_length = 0; // Can never go below zero. + + // Metrics. + std::atomic metric_input_queue_safe_length_frames{1}; +}; + +class Mixer { +public: + // The surface format is used for offscreen destinations for OpenGL contexts we need. + Mixer(const QSurfaceFormat &format, unsigned num_cards); + ~Mixer(); + void start(); + void quit(); + + void transition_clicked(int transition_num); + void channel_clicked(int preview_num); + + enum Output { + OUTPUT_LIVE = 0, + OUTPUT_PREVIEW, + OUTPUT_INPUT0, // 1, 2, 3, up to 15 follow numerically. + NUM_OUTPUTS = 18 + }; + + struct DisplayFrame { + // The chain for rendering this frame. To render a display frame, + // first wait for , then call + // to wire up all the inputs, and then finally call + // chain->render_to_screen() or similar. + movit::EffectChain *chain; + std::function setup_chain; + + // Asserted when all the inputs are ready; you cannot render the chain + // before this. + RefCountedGLsync ready_fence; + + // Holds on to all the input frames needed for this display frame, + // so they are not released while still rendering. + std::vector input_frames; + + // Textures that should be released back to the resource pool + // when this frame disappears, if any. + // TODO: Refcount these as well? + std::vector temp_textures; + }; + // Implicitly frees the previous one if there's a new frame available. + bool get_display_frame(Output output, DisplayFrame *frame) { + return output_channel[output].get_display_frame(frame); + } + + // NOTE: Callbacks will be called with a mutex held, so you should probably + // not do real work in them. + typedef std::function new_frame_ready_callback_t; + void add_frame_ready_callback(Output output, void *key, new_frame_ready_callback_t callback) + { + output_channel[output].add_frame_ready_callback(key, callback); + } + + void remove_frame_ready_callback(Output output, void *key) + { + output_channel[output].remove_frame_ready_callback(key); + } + + // TODO: Should this really be per-channel? Shouldn't it just be called for e.g. the live output? + typedef std::function &)> transition_names_updated_callback_t; + void set_transition_names_updated_callback(Output output, transition_names_updated_callback_t callback) + { + output_channel[output].set_transition_names_updated_callback(callback); + } + + typedef std::function name_updated_callback_t; + void set_name_updated_callback(Output output, name_updated_callback_t callback) + { + output_channel[output].set_name_updated_callback(callback); + } + + typedef std::function color_updated_callback_t; + void set_color_updated_callback(Output output, color_updated_callback_t callback) + { + output_channel[output].set_color_updated_callback(callback); + } + + std::vector get_transition_names() + { + return theme->get_transition_names(pts()); + } + + unsigned get_num_channels() const + { + return theme->get_num_channels(); + } + + std::string get_channel_name(unsigned channel) const + { + return theme->get_channel_name(channel); + } + + std::string get_channel_color(unsigned channel) const + { + return theme->get_channel_color(channel); + } + + int get_channel_signal(unsigned channel) const + { + return theme->get_channel_signal(channel); + } + + int map_signal(unsigned channel) + { + return theme->map_signal(channel); + } + + unsigned get_master_clock() const + { + return master_clock_channel; + } + + void set_master_clock(unsigned channel) + { + master_clock_channel = channel; + } + + void set_signal_mapping(int signal, int card) + { + return theme->set_signal_mapping(signal, card); + } + + YCbCrInterpretation get_input_ycbcr_interpretation(unsigned card_index) const; + void set_input_ycbcr_interpretation(unsigned card_index, const YCbCrInterpretation &interpretation); + + bool get_supports_set_wb(unsigned channel) const + { + return theme->get_supports_set_wb(channel); + } + + void set_wb(unsigned channel, double r, double g, double b) const + { + theme->set_wb(channel, r, g, b); + } + + // Note: You can also get this through the global variable global_audio_mixer. + AudioMixer *get_audio_mixer() { return audio_mixer.get(); } + const AudioMixer *get_audio_mixer() const { return audio_mixer.get(); } + + void schedule_cut() + { + should_cut = true; + } + + unsigned get_num_cards() const { return num_cards; } + + std::string get_card_description(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_description(); + } + + // The difference between this and the previous function is that if a card + // is used as the current output, get_card_description() will return the + // fake card that's replacing it for input, whereas this function will return + // the card's actual name. + std::string get_output_card_description(unsigned card_index) const { + assert(card_can_be_used_as_output(card_index)); + assert(card_index < num_cards); + if (cards[card_index].parked_capture) { + return cards[card_index].parked_capture->get_description(); + } else { + return cards[card_index].capture->get_description(); + } + } + + bool card_can_be_used_as_output(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].output != nullptr; + } + + bool card_is_ffmpeg(unsigned card_index) const { + assert(card_index < num_cards + num_video_inputs); + return cards[card_index].type == CardType::FFMPEG_INPUT; + } + + std::map get_available_video_modes(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_video_modes(); + } + + uint32_t get_current_video_mode(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_video_mode(); + } + + void set_video_mode(unsigned card_index, uint32_t mode) { + assert(card_index < num_cards); + cards[card_index].capture->set_video_mode(mode); + } + + void start_mode_scanning(unsigned card_index); + + std::map get_available_video_inputs(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_video_inputs(); + } + + uint32_t get_current_video_input(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_video_input(); + } + + void set_video_input(unsigned card_index, uint32_t input) { + assert(card_index < num_cards); + cards[card_index].capture->set_video_input(input); + } + + std::map get_available_audio_inputs(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_audio_inputs(); + } + + uint32_t get_current_audio_input(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_audio_input(); + } + + void set_audio_input(unsigned card_index, uint32_t input) { + assert(card_index < num_cards); + cards[card_index].capture->set_audio_input(input); + } + + std::string get_ffmpeg_filename(unsigned card_index) const; + + void set_ffmpeg_filename(unsigned card_index, const std::string &filename); + + void change_x264_bitrate(unsigned rate_kbit) { + video_encoder->change_x264_bitrate(rate_kbit); + } + + int get_output_card_index() const { // -1 = no output, just stream. + return desired_output_card_index; + } + + void set_output_card(int card_index) { // -1 = no output, just stream. + desired_output_card_index = card_index; + } + + std::map get_available_output_video_modes() const; + + uint32_t get_output_video_mode() const { + return desired_output_video_mode; + } + + void set_output_video_mode(uint32_t mode) { + desired_output_video_mode = mode; + } + + void set_display_timecode_in_stream(bool enable) { + display_timecode_in_stream = enable; + } + + void set_display_timecode_on_stdout(bool enable) { + display_timecode_on_stdout = enable; + } + + int64_t get_num_connected_clients() const { + return httpd.get_num_connected_clients(); + } + + std::vector get_theme_menu() { return theme->get_theme_menu(); } + + void theme_menu_entry_clicked(int lua_ref) { return theme->theme_menu_entry_clicked(lua_ref); } + + void set_theme_menu_callback(std::function callback) + { + theme->set_theme_menu_callback(callback); + } + + void wait_for_next_frame(); + +private: + struct CaptureCard; + + enum class CardType { + LIVE_CARD, + FAKE_CAPTURE, + FFMPEG_INPUT, + CEF_INPUT, + }; + void configure_card(unsigned card_index, bmusb::CaptureInterface *capture, CardType card_type, DeckLinkOutput *output); + void set_output_card_internal(int card_index); // Should only be called from the mixer thread. + void bm_frame(unsigned card_index, uint16_t timecode, + bmusb::FrameAllocator::Frame video_frame, size_t video_offset, bmusb::VideoFormat video_format, + bmusb::FrameAllocator::Frame audio_frame, size_t audio_offset, bmusb::AudioFormat audio_format); + void bm_hotplug_add(libusb_device *dev); + void bm_hotplug_remove(unsigned card_index); + void place_rectangle(movit::Effect *resample_effect, movit::Effect *padding_effect, float x0, float y0, float x1, float y1); + void thread_func(); + void handle_hotplugged_cards(); + void schedule_audio_resampling_tasks(unsigned dropped_frames, int num_samples_per_frame, int length_per_frame, bool is_preroll, std::chrono::steady_clock::time_point frame_timestamp); + std::string get_timecode_text() const; + void render_one_frame(int64_t duration); + void audio_thread_func(); + void release_display_frame(DisplayFrame *frame); + double pts() { return double(pts_int) / TIMEBASE; } + void trim_queue(CaptureCard *card, size_t safe_queue_length); + std::pair get_channels_json(); + std::pair get_channel_color_http(unsigned channel_idx); + + HTTPD httpd; + unsigned num_cards, num_video_inputs, num_html_inputs = 0; + + QSurface *mixer_surface, *h264_encoder_surface, *decklink_output_surface; + std::unique_ptr resource_pool; + std::unique_ptr theme; + std::atomic audio_source_channel{0}; + std::atomic master_clock_channel{0}; // Gets overridden by if set. + int output_card_index = -1; // -1 for none. + uint32_t output_video_mode = -1; + + // The mechanics of changing the output card and modes are so intricately connected + // with the work the mixer thread is doing. Thus, we don't change it directly, + // we just set this variable instead, which signals to the mixer thread that + // it should do the change before the next frame. This simplifies locking + // considerations immensely. + std::atomic desired_output_card_index{-1}; + std::atomic desired_output_video_mode{0}; + + std::unique_ptr display_chain; + std::unique_ptr chroma_subsampler; + std::unique_ptr v210_converter; + std::unique_ptr video_encoder; + + std::unique_ptr timecode_renderer; + std::atomic display_timecode_in_stream{false}; + std::atomic display_timecode_on_stdout{false}; + + // Effects part of . Owned by . + movit::YCbCrInput *display_input; + + int64_t pts_int = 0; // In TIMEBASE units. + + mutable std::mutex frame_num_mutex; + std::condition_variable frame_num_updated; + unsigned frame_num = 0; // Under . + + // Accumulated errors in number of 1/TIMEBASE audio samples. If OUTPUT_FREQUENCY divided by + // frame rate is integer, will always stay zero. + unsigned fractional_samples = 0; + + mutable std::mutex card_mutex; + bool has_bmusb_thread = false; + struct CaptureCard { + std::unique_ptr capture; + bool is_fake_capture; + CardType type; + std::unique_ptr output; + + // CEF only delivers frames when it actually has a change. + // If we trim the queue for latency reasons, we could thus + // end up in a situation trimming a frame that was meant to + // be displayed for a long time, which is really suboptimal. + // Thus, if we drop the last frame we have, may_have_dropped_last_frame + // is set to true, and the next starvation event will trigger + // us requestin a CEF repaint. + bool is_cef_capture, may_have_dropped_last_frame = false; + + // If this card is used for output (ie., output_card_index points to it), + // it cannot simultaneously be uesd for capture, so gets replaced + // by a FakeCapture. However, since reconstructing the real capture object + // with all its state can be annoying, it is not being deleted, just stopped + // and moved here. + std::unique_ptr parked_capture; + + std::unique_ptr frame_allocator; + + // Stuff for the OpenGL context (for texture uploading). + QSurface *surface = nullptr; + + struct NewFrame { + RefCountedFrame frame; + int64_t length; // In TIMEBASE units. + bool interlaced; + unsigned field; // Which field (0 or 1) of the frame to use. Always 0 for progressive. + std::function upload_func; // Needs to be called to actually upload the texture to OpenGL. + unsigned dropped_frames = 0; // Number of dropped frames before this one. + std::chrono::steady_clock::time_point received_timestamp = std::chrono::steady_clock::time_point::min(); + }; + std::deque new_frames; + std::condition_variable new_frames_changed; // Set whenever new_frames is changed. + + QueueLengthPolicy queue_length_policy; // Refers to the "new_frames" queue. + + int last_timecode = -1; // Unwrapped. + + JitterHistory jitter_history; + + // Metrics. + std::vector> labels; + std::atomic metric_input_received_frames{0}; + std::atomic metric_input_duped_frames{0}; + std::atomic metric_input_dropped_frames_jitter{0}; + std::atomic metric_input_dropped_frames_error{0}; + std::atomic metric_input_resets{0}; + std::atomic metric_input_queue_length_frames{0}; + + std::atomic metric_input_has_signal_bool{-1}; + std::atomic metric_input_is_connected_bool{-1}; + std::atomic metric_input_interlaced_bool{-1}; + std::atomic metric_input_width_pixels{-1}; + std::atomic metric_input_height_pixels{-1}; + std::atomic metric_input_frame_rate_nom{-1}; + std::atomic metric_input_frame_rate_den{-1}; + std::atomic metric_input_sample_rate_hz{-1}; + }; + JitterHistory output_jitter_history; + CaptureCard cards[MAX_VIDEO_CARDS]; // Protected by . + YCbCrInterpretation ycbcr_interpretation[MAX_VIDEO_CARDS]; // Protected by . + std::unique_ptr audio_mixer; // Same as global_audio_mixer (see audio_mixer.h). + bool input_card_is_master_clock(unsigned card_index, unsigned master_card_index) const; + struct OutputFrameInfo { + int dropped_frames; // Since last frame. + int num_samples; // Audio samples needed for this output frame. + int64_t frame_duration; // In TIMEBASE units. + bool is_preroll; + std::chrono::steady_clock::time_point frame_timestamp; + }; + OutputFrameInfo get_one_frame_from_each_card(unsigned master_card_index, bool master_card_is_output, CaptureCard::NewFrame new_frames[MAX_VIDEO_CARDS], bool has_new_frame[MAX_VIDEO_CARDS]); + + InputState input_state; + + // Cards we have been noticed about being hotplugged, but haven't tried adding yet. + // Protected by its own mutex. + std::mutex hotplug_mutex; + std::vector hotplugged_cards; + + class OutputChannel { + public: + ~OutputChannel(); + void output_frame(DisplayFrame &&frame); + bool get_display_frame(DisplayFrame *frame); + void add_frame_ready_callback(void *key, new_frame_ready_callback_t callback); + void remove_frame_ready_callback(void *key); + void set_transition_names_updated_callback(transition_names_updated_callback_t callback); + void set_name_updated_callback(name_updated_callback_t callback); + void set_color_updated_callback(color_updated_callback_t callback); + + private: + friend class Mixer; + + unsigned channel; + Mixer *parent = nullptr; // Not owned. + std::mutex frame_mutex; + DisplayFrame current_frame, ready_frame; // protected by + bool has_current_frame = false, has_ready_frame = false; // protected by + std::map new_frame_ready_callbacks; // protected by + transition_names_updated_callback_t transition_names_updated_callback; + name_updated_callback_t name_updated_callback; + color_updated_callback_t color_updated_callback; + + std::vector last_transition_names; + std::string last_name, last_color; + }; + OutputChannel output_channel[NUM_OUTPUTS]; + + std::thread mixer_thread; + std::thread audio_thread; + std::atomic should_quit{false}; + std::atomic should_cut{false}; + + std::unique_ptr alsa; + + struct AudioTask { + int64_t pts_int; + int num_samples; + bool adjust_rate; + std::chrono::steady_clock::time_point frame_timestamp; + }; + std::mutex audio_mutex; + std::condition_variable audio_task_queue_changed; + std::queue audio_task_queue; // Under audio_mutex. + + // For mode scanning. + bool is_mode_scanning[MAX_VIDEO_CARDS]{ false }; + std::vector mode_scanlist[MAX_VIDEO_CARDS]; + unsigned mode_scanlist_index[MAX_VIDEO_CARDS]{ 0 }; + std::chrono::steady_clock::time_point last_mode_scan_change[MAX_VIDEO_CARDS]; +}; + +extern Mixer *global_mixer; + +#endif // !defined(_MIXER_H) diff --git a/nageru/mux.cpp b/nageru/mux.cpp new file mode 100644 index 0000000..b1b9db6 --- /dev/null +++ b/nageru/mux.cpp @@ -0,0 +1,275 @@ +#include "mux.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +} + +#include "defs.h" +#include "flags.h" +#include "metrics.h" +#include "timebase.h" + +using namespace std; + +struct PacketBefore { + PacketBefore(const AVFormatContext *ctx) : ctx(ctx) {} + + bool operator() (const Mux::QueuedPacket &a_qp, const Mux::QueuedPacket &b_qp) const { + const AVPacket *a = a_qp.pkt; + const AVPacket *b = b_qp.pkt; + int64_t a_dts = (a->dts == AV_NOPTS_VALUE ? a->pts : a->dts); + int64_t b_dts = (b->dts == AV_NOPTS_VALUE ? b->pts : b->dts); + AVRational a_timebase = ctx->streams[a->stream_index]->time_base; + AVRational b_timebase = ctx->streams[b->stream_index]->time_base; + if (av_compare_ts(a_dts, a_timebase, b_dts, b_timebase) != 0) { + return av_compare_ts(a_dts, a_timebase, b_dts, b_timebase) < 0; + } else { + return av_compare_ts(a->pts, a_timebase, b->pts, b_timebase) < 0; + } + } + + const AVFormatContext * const ctx; +}; + +Mux::Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const string &video_extradata, const AVCodecParameters *audio_codecpar, int time_base, std::function write_callback, WriteStrategy write_strategy, const vector &metrics) + : write_strategy(write_strategy), avctx(avctx), write_callback(write_callback), metrics(metrics) +{ + avstream_video = avformat_new_stream(avctx, nullptr); + if (avstream_video == nullptr) { + fprintf(stderr, "avformat_new_stream() failed\n"); + exit(1); + } + avstream_video->time_base = AVRational{1, time_base}; + avstream_video->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + if (video_codec == CODEC_H264) { + avstream_video->codecpar->codec_id = AV_CODEC_ID_H264; + } else { + assert(video_codec == CODEC_NV12); + avstream_video->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO; + avstream_video->codecpar->codec_tag = avcodec_pix_fmt_to_codec_tag(AV_PIX_FMT_NV12); + } + avstream_video->codecpar->width = width; + avstream_video->codecpar->height = height; + + // Colorspace details. Closely correspond to settings in EffectChain_finalize, + // as noted in each comment. + // Note that the H.264 stream also contains this information and depending on the + // mux, this might simply get ignored. See sps_rbsp(). + // Note that there's no way to change this per-frame as the H.264 stream + // would like to be able to. + avstream_video->codecpar->color_primaries = AVCOL_PRI_BT709; // RGB colorspace (inout_format.color_space). + avstream_video->codecpar->color_trc = AVCOL_TRC_IEC61966_2_1; // Gamma curve (inout_format.gamma_curve). + // YUV colorspace (output_ycbcr_format.luma_coefficients). + if (global_flags.ycbcr_rec709_coefficients) { + avstream_video->codecpar->color_space = AVCOL_SPC_BT709; + } else { + avstream_video->codecpar->color_space = AVCOL_SPC_SMPTE170M; + } + avstream_video->codecpar->color_range = AVCOL_RANGE_MPEG; // Full vs. limited range (output_ycbcr_format.full_range). + avstream_video->codecpar->chroma_location = AVCHROMA_LOC_LEFT; // Chroma sample location. See chroma_offset_0[] in Mixer::subsample_chroma(). + avstream_video->codecpar->field_order = AV_FIELD_PROGRESSIVE; + + if (!video_extradata.empty()) { + avstream_video->codecpar->extradata = (uint8_t *)av_malloc(video_extradata.size()); + avstream_video->codecpar->extradata_size = video_extradata.size(); + memcpy(avstream_video->codecpar->extradata, video_extradata.data(), video_extradata.size()); + } + + avstream_audio = avformat_new_stream(avctx, nullptr); + if (avstream_audio == nullptr) { + fprintf(stderr, "avformat_new_stream() failed\n"); + exit(1); + } + avstream_audio->time_base = AVRational{1, time_base}; + if (avcodec_parameters_copy(avstream_audio->codecpar, audio_codecpar) < 0) { + fprintf(stderr, "avcodec_parameters_copy() failed\n"); + exit(1); + } + + AVDictionary *options = NULL; + vector> opts = MUX_OPTS; + for (pair opt : opts) { + av_dict_set(&options, opt.first.c_str(), opt.second.c_str(), 0); + } + if (avformat_write_header(avctx, &options) < 0) { + fprintf(stderr, "avformat_write_header() failed\n"); + exit(1); + } + for (MuxMetrics *metric : metrics) { + metric->metric_written_bytes += avctx->pb->pos; + } + + // Make sure the header is written before the constructor exits. + avio_flush(avctx->pb); + + if (write_strategy == WRITE_BACKGROUND) { + writer_thread = thread(&Mux::thread_func, this); + } +} + +Mux::~Mux() +{ + assert(plug_count == 0); + if (write_strategy == WRITE_BACKGROUND) { + writer_thread_should_quit = true; + packet_queue_ready.notify_all(); + writer_thread.join(); + } + int64_t old_pos = avctx->pb->pos; + av_write_trailer(avctx); + for (MuxMetrics *metric : metrics) { + metric->metric_written_bytes += avctx->pb->pos - old_pos; + } + + if (!(avctx->oformat->flags & AVFMT_NOFILE) && + !(avctx->flags & AVFMT_FLAG_CUSTOM_IO)) { + avio_closep(&avctx->pb); + } + avformat_free_context(avctx); +} + +void Mux::add_packet(const AVPacket &pkt, int64_t pts, int64_t dts, AVRational timebase, int stream_index_override) +{ + AVPacket pkt_copy; + av_init_packet(&pkt_copy); + if (av_packet_ref(&pkt_copy, &pkt) < 0) { + fprintf(stderr, "av_copy_packet() failed\n"); + exit(1); + } + if (stream_index_override != -1) { + pkt_copy.stream_index = stream_index_override; + } + if (pkt_copy.stream_index == 0) { + pkt_copy.pts = av_rescale_q(pts, timebase, avstream_video->time_base); + pkt_copy.dts = av_rescale_q(dts, timebase, avstream_video->time_base); + pkt_copy.duration = av_rescale_q(pkt.duration, timebase, avstream_video->time_base); + } else if (pkt_copy.stream_index == 1) { + pkt_copy.pts = av_rescale_q(pts, timebase, avstream_audio->time_base); + pkt_copy.dts = av_rescale_q(dts, timebase, avstream_audio->time_base); + pkt_copy.duration = av_rescale_q(pkt.duration, timebase, avstream_audio->time_base); + } else { + assert(false); + } + + { + lock_guard lock(mu); + if (write_strategy == WriteStrategy::WRITE_BACKGROUND) { + packet_queue.push_back(QueuedPacket{ av_packet_clone(&pkt_copy), pts }); + if (plug_count == 0) packet_queue_ready.notify_all(); + } else if (plug_count > 0) { + packet_queue.push_back(QueuedPacket{ av_packet_clone(&pkt_copy), pts }); + } else { + write_packet_or_die(pkt_copy, pts); + } + } + + av_packet_unref(&pkt_copy); +} + +void Mux::write_packet_or_die(const AVPacket &pkt, int64_t unscaled_pts) +{ + for (MuxMetrics *metric : metrics) { + if (pkt.stream_index == 0) { + metric->metric_video_bytes += pkt.size; + } else if (pkt.stream_index == 1) { + metric->metric_audio_bytes += pkt.size; + } else { + assert(false); + } + } + int64_t old_pos = avctx->pb->pos; + if (av_interleaved_write_frame(avctx, const_cast(&pkt)) < 0) { + fprintf(stderr, "av_interleaved_write_frame() failed\n"); + exit(1); + } + avio_flush(avctx->pb); + for (MuxMetrics *metric : metrics) { + metric->metric_written_bytes += avctx->pb->pos - old_pos; + } + + if (pkt.stream_index == 0 && write_callback != nullptr) { + write_callback(unscaled_pts); + } +} + +void Mux::plug() +{ + lock_guard lock(mu); + ++plug_count; +} + +void Mux::unplug() +{ + lock_guard lock(mu); + if (--plug_count > 0) { + return; + } + assert(plug_count >= 0); + + sort(packet_queue.begin(), packet_queue.end(), PacketBefore(avctx)); + + if (write_strategy == WRITE_BACKGROUND) { + packet_queue_ready.notify_all(); + } else { + for (QueuedPacket &qp : packet_queue) { + write_packet_or_die(*qp.pkt, qp.unscaled_pts); + av_packet_free(&qp.pkt); + } + packet_queue.clear(); + } +} + +void Mux::thread_func() +{ + unique_lock lock(mu); + for ( ;; ) { + packet_queue_ready.wait(lock, [this]() { + return writer_thread_should_quit || (!packet_queue.empty() && plug_count == 0); + }); + if (writer_thread_should_quit && packet_queue.empty()) { + // All done. + break; + } + + assert(!packet_queue.empty() && plug_count == 0); + vector packets; + swap(packets, packet_queue); + + lock.unlock(); + for (QueuedPacket &qp : packets) { + write_packet_or_die(*qp.pkt, qp.unscaled_pts); + av_packet_free(&qp.pkt); + } + lock.lock(); + } +} + +void MuxMetrics::init(const vector> &labels) +{ + vector> labels_video = labels; + labels_video.emplace_back("stream", "video"); + global_metrics.add("mux_stream_bytes", labels_video, &metric_video_bytes); + + vector> labels_audio = labels; + labels_audio.emplace_back("stream", "audio"); + global_metrics.add("mux_stream_bytes", labels_audio, &metric_audio_bytes); + + global_metrics.add("mux_written_bytes", labels, &metric_written_bytes); +} diff --git a/nageru/mux.h b/nageru/mux.h new file mode 100644 index 0000000..9614bff --- /dev/null +++ b/nageru/mux.h @@ -0,0 +1,111 @@ +#ifndef _MUX_H +#define _MUX_H 1 + +// Wrapper around an AVFormat mux. + +extern "C" { +#include +#include +} + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "timebase.h" + +struct MuxMetrics { + // “written” will usually be equal video + audio + mux overhead, + // except that there could be buffered packets that count in audio or video + // but not yet in written. + std::atomic metric_video_bytes{0}, metric_audio_bytes{0}, metric_written_bytes{0}; + + // Registers in global_metrics. + void init(const std::vector> &labels); + + void reset() + { + metric_video_bytes = 0; + metric_audio_bytes = 0; + metric_written_bytes = 0; + } +}; + +class Mux { +public: + enum Codec { + CODEC_H264, + CODEC_NV12, // Uncompressed 4:2:0. + }; + enum WriteStrategy { + // add_packet() will write the packet immediately, unless plugged. + WRITE_FOREGROUND, + + // All writes will happen on a separate thread, so add_packet() + // won't block. Use this if writing to a file and you might be + // holding a mutex (because blocking I/O with a mutex held is + // not good). Note that this will clone every packet, so it has + // higher overhead. + WRITE_BACKGROUND, + }; + + // Takes ownership of avctx. will be called every time + // a write has been made to the video stream (id 0), with the pts of + // the just-written frame. (write_callback can be nullptr.) + // Does not take ownership of ; elements in there, if any, + // will be added to. + Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const std::string &video_extradata, const AVCodecParameters *audio_codecpar, int time_base, std::function write_callback, WriteStrategy write_strategy, const std::vector &metrics); + ~Mux(); + void add_packet(const AVPacket &pkt, int64_t pts, int64_t dts, AVRational timebase = { 1, TIMEBASE }, int stream_index_override = -1); + + // As long as the mux is plugged, it will not actually write anything to disk, + // just queue the packets. Once it is unplugged, the packets are reordered by pts + // and written. This is primarily useful if you might have two different encoders + // writing to the mux at the same time (because one is shutting down), so that + // pts might otherwise come out-of-order. + // + // You can plug and unplug multiple times; only when the plug count reaches zero, + // something will actually happen. + void plug(); + void unplug(); + +private: + // If write_strategy == WRITE_FOREGORUND, Must be called with held. + void write_packet_or_die(const AVPacket &pkt, int64_t unscaled_pts); + void thread_func(); + + WriteStrategy write_strategy; + + std::mutex mu; + + // These are only in use if write_strategy == WRITE_BACKGROUND. + std::atomic writer_thread_should_quit{false}; + std::thread writer_thread; + + AVFormatContext *avctx; // Protected by , iff write_strategy == WRITE_BACKGROUND. + int plug_count = 0; // Protected by . + + // Protected by . If write_strategy == WRITE_FOREGROUND, + // this is only in use when plugging. + struct QueuedPacket { + AVPacket *pkt; + int64_t unscaled_pts; + }; + std::vector packet_queue; + std::condition_variable packet_queue_ready; + + AVStream *avstream_video, *avstream_audio; + + std::function write_callback; + std::vector metrics; + + friend struct PacketBefore; +}; + +#endif // !defined(_MUX_H) diff --git a/nageru/nageru_cef_app.cpp b/nageru/nageru_cef_app.cpp new file mode 100644 index 0000000..2e64cee --- /dev/null +++ b/nageru/nageru_cef_app.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include + +#include "nageru_cef_app.h" + +using namespace std; + +void NageruCefApp::OnBeforeCommandLineProcessing( + const CefString& process_type, + CefRefPtr command_line) +{ + command_line->AppendSwitch("disable-gpu"); + command_line->AppendSwitch("disable-gpu-compositing"); + command_line->AppendSwitch("enable-begin-frame-scheduling"); +} + +void NageruCefApp::initialize_cef() +{ + unique_lock lock(cef_mutex); + if (cef_thread_refcount++ == 0) { + cef_thread = thread(&NageruCefApp::cef_thread_func, this); + } + cef_initialized_cond.wait(lock, [this]{ return cef_initialized; }); +} + +void NageruCefApp::close_browser(CefRefPtr browser) +{ + unique_lock lock(cef_mutex); + browser->GetHost()->CloseBrowser(/*force_close=*/true); +} + +void NageruCefApp::unref_cef() +{ + unique_lock lock(cef_mutex); + if (--cef_thread_refcount == 0) { + CefPostTask(TID_UI, new CEFTaskAdapter(&CefQuitMessageLoop)); + lock.unlock(); + cef_thread.join(); + } +} + +void NageruCefApp::cef_thread_func() +{ + CefMainArgs main_args; + CefSettings settings; + //settings.log_severity = LOGSEVERITY_VERBOSE; + settings.windowless_rendering_enabled = true; + settings.no_sandbox = true; + settings.command_line_args_disabled = false; + CefInitialize(main_args, settings, this, nullptr); + + { + lock_guard lock(cef_mutex); + cef_initialized = true; + } + cef_initialized_cond.notify_all(); + + CefRunMessageLoop(); + + CefShutdown(); +} + diff --git a/nageru/nageru_cef_app.h b/nageru/nageru_cef_app.h new file mode 100644 index 0000000..7b8969b --- /dev/null +++ b/nageru/nageru_cef_app.h @@ -0,0 +1,93 @@ +#ifndef _NAGERU_CEF_APP_H +#define _NAGERU_CEF_APP_H 1 + +// NageruCefApp deals with global state around CEF, in particular the global +// CEF event loop. CEF is pretty picky about which threads everything runs on; +// in particular, the documentation says CefExecute, CefInitialize and +// CefRunMessageLoop must all be on the main thread (ie., the first thread +// created). However, Qt wants to run _its_ event loop on this thread, too, +// and integrating the two has proved problematic (see also the comment in +// main.cpp). It seems that as long as you don't have two GLib loops running, +// it's completely fine in practice to have a separate thread for the main loop +// (running CefInitialize, CefRunMessageLoop, and finally CefDestroy). +// Many other tasks (like most things related to interacting with browsers) +// have to be run from the message loop, but that's fine; CEF gives us tools +// to post tasks to it. + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// Takes in arbitrary lambdas and converts them to something CefPostTask() will accept. +class CEFTaskAdapter : public CefTask +{ +public: + CEFTaskAdapter(const std::function&& func) + : func(std::move(func)) {} + void Execute() override { func(); } + +private: + std::function func; + + IMPLEMENT_REFCOUNTING(CEFTaskAdapter); +}; + +// Runs and stops the CEF event loop, and also makes some startup tasks. +class NageruCefApp : public CefApp, public CefRenderProcessHandler, public CefBrowserProcessHandler { +public: + NageruCefApp() {} + + // Starts up the CEF main loop if it does not already run, and blocks until + // CEF is properly initialized. You can call initialize_ref() multiple times, + // which will then increase the refcount. + void initialize_cef(); + + // If the refcount goes to zero, shut down the main loop and uninitialize CEF. + void unref_cef(); + + // Closes the given browser, and blocks until it is done closing. + // + // NOTE: We can't call unref_cef() from close_browser(), since + // CefRefPtr does not support move semantics, so it would have a + // refcount of either zero or two going into close_browser (not one, + // as it should). The latter means the caller would hold on to an extra + // reference to the browser (which triggers an assert failure), and the + // former would mean that the browser gets deleted before it's closed. + void close_browser(CefRefPtr browser); + + CefRefPtr GetRenderProcessHandler() override + { + return this; + } + + CefRefPtr GetBrowserProcessHandler() override + { + return this; + } + + void OnBeforeCommandLineProcessing(const CefString& process_type, CefRefPtr command_line) override; + +private: + void cef_thread_func(); + + std::thread cef_thread; + std::mutex cef_mutex; + int cef_thread_refcount = 0; // Under . + bool cef_initialized = false; // Under . + std::condition_variable cef_initialized_cond; + + IMPLEMENT_REFCOUNTING(NageruCefApp); +}; + +#endif // !defined(_NAGERU_CEF_APP_H) diff --git a/nageru/nonlinear_fader.cpp b/nageru/nonlinear_fader.cpp new file mode 100644 index 0000000..06a929c --- /dev/null +++ b/nageru/nonlinear_fader.cpp @@ -0,0 +1,108 @@ +#include "nonlinear_fader.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "piecewise_interpolator.h" + +class QPaintEvent; +class QWidget; + +using namespace std; + +namespace { + +PiecewiseInterpolator interpolator({ + // The main area is from +6 to -12 dB (18 dB), and we use half the slider range for it. + // Adjust slightly so that the MIDI controller value of 106 becomes exactly 0.0 dB + // (cf. map_controller_to_float()); otherwise, we'd miss ever so slightly, which is + // really frustrating. + { 6.0, 1.0 }, + { -12.0, 1.0 - (1.0 - 106.5/127.0) * 3.0 }, // About 0.492. + + // -12 to -21 is half the range (9 dB). Halve. + { -21.0, 0.325 }, + + // -21 to -30 (9 dB) gets the same range as the previous one. + { -30.0, 0.25 }, + + // -30 to -48 (18 dB) gets half of half. + { -48.0, 0.125 }, + + // -48 to -84 (36 dB) gets half of half of half. + { -84.0, 0.0 }, +}); + +} // namespace + +NonLinearFader::NonLinearFader(QWidget *parent) + : QSlider(parent) +{ + update_slider_position(); +} + +void NonLinearFader::setDbValue(double db) +{ + db_value = db; + update_slider_position(); + emit dbValueChanged(db); +} + +void NonLinearFader::paintEvent(QPaintEvent *event) +{ + QStyleOptionSlider opt; + this->initStyleOption(&opt); + QRect gr = this->style()->subControlRect(QStyle::CC_Slider, &opt, QStyle::SC_SliderGroove, this); + QRect sr = this->style()->subControlRect(QStyle::CC_Slider, &opt, QStyle::SC_SliderHandle, this); + + // FIXME: Where does the slider_length / 2 come from? I can't really find it + // in the Qt code, but it seems to match up with reality. + int slider_length = sr.height(); + int slider_max = gr.top() + (slider_length / 2); + int slider_min = gr.bottom() + (slider_length / 2) - slider_length + 1; + + QPainter p(this); + + // Draw some ticks every 6 dB. + // FIXME: Find a way to make the slider wider, so that we have more space for tickmarks + // and some dB numbering. + int x_margin = 5; + p.setPen(Qt::darkGray); + for (int db = -84; db <= 6; db += 6) { + int y = slider_min + lrint(interpolator.db_to_fraction(db) * (slider_max - slider_min)); + p.drawLine(QPoint(0, y), QPoint(gr.left() - x_margin, y)); + p.drawLine(QPoint(gr.right() + x_margin, y), QPoint(width() - 1, y)); + } + + QSlider::paintEvent(event); +} + +void NonLinearFader::sliderChange(SliderChange change) +{ + QSlider::sliderChange(change); + if (change == QAbstractSlider::SliderValueChange && !inhibit_updates) { + if (value() == 0) { + db_value = -HUGE_VAL; + } else { + double frac = double(value() - minimum()) / (maximum() - minimum()); + db_value = interpolator.fraction_to_db(frac); + } + emit dbValueChanged(db_value); + } +} + +void NonLinearFader::update_slider_position() +{ + inhibit_updates = true; + double val = interpolator.db_to_fraction(db_value) * (maximum() - minimum()) + minimum(); + setValue(lrint(val)); + inhibit_updates = false; +} diff --git a/nageru/nonlinear_fader.h b/nageru/nonlinear_fader.h new file mode 100644 index 0000000..ce72852 --- /dev/null +++ b/nageru/nonlinear_fader.h @@ -0,0 +1,33 @@ +#ifndef _NONLINEAR_FADER_H +#define _NONLINEAR_FADER_H 1 + +#include +#include +#include + +class QObject; +class QPaintEvent; +class QWidget; + +class NonLinearFader : public QSlider { + Q_OBJECT + +public: + NonLinearFader(QWidget *parent); + void setDbValue(double db); + +signals: + void dbValueChanged(double db); + +protected: + void paintEvent(QPaintEvent *event) override; + void sliderChange(SliderChange change) override; + +private: + void update_slider_position(); + + bool inhibit_updates = false; + double db_value = 0.0; +}; + +#endif // !defined(_NONLINEAR_FADER_H) diff --git a/nageru/pbo_frame_allocator.cpp b/nageru/pbo_frame_allocator.cpp new file mode 100644 index 0000000..ea17f12 --- /dev/null +++ b/nageru/pbo_frame_allocator.cpp @@ -0,0 +1,312 @@ +#include "pbo_frame_allocator.h" + +#include +#include +#include +#include +#include +#include + +#include "flags.h" +#include "v210_converter.h" + +using namespace std; + +namespace { + +void set_clamp_to_edge() +{ + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); +} + +} // namespace + +PBOFrameAllocator::PBOFrameAllocator(bmusb::PixelFormat pixel_format, size_t frame_size, GLuint width, GLuint height, size_t num_queued_frames, GLenum buffer, GLenum permissions, GLenum map_bits) + : pixel_format(pixel_format), buffer(buffer) +{ + userdata.reset(new Userdata[num_queued_frames]); + for (size_t i = 0; i < num_queued_frames; ++i) { + init_frame(i, frame_size, width, height, permissions, map_bits); + } + glBindBuffer(buffer, 0); + check_error(); + glBindTexture(GL_TEXTURE_2D, 0); + check_error(); +} + +void PBOFrameAllocator::init_frame(size_t frame_idx, size_t frame_size, GLuint width, GLuint height, GLenum permissions, GLenum map_bits) +{ + GLuint pbo; + glGenBuffers(1, &pbo); + check_error(); + glBindBuffer(buffer, pbo); + check_error(); + glBufferStorage(buffer, frame_size, nullptr, permissions | GL_MAP_PERSISTENT_BIT); + check_error(); + + Frame frame; + frame.data = (uint8_t *)glMapBufferRange(buffer, 0, frame_size, permissions | map_bits | GL_MAP_PERSISTENT_BIT); + frame.data2 = frame.data + frame_size / 2; + check_error(); + frame.size = frame_size; + frame.userdata = &userdata[frame_idx]; + userdata[frame_idx].pbo = pbo; + userdata[frame_idx].pixel_format = pixel_format; + frame.owner = this; + + // For 8-bit non-planar Y'CbCr, we ask the driver to split Y' and Cb/Cr + // into separate textures. For 10-bit, the input format (v210) + // is complicated enough that we need to interpolate up to 4:4:4, + // which we do in a compute shader ourselves. For BGRA, the data + // is already 4:4:4:4. + frame.interleaved = (pixel_format == bmusb::PixelFormat_8BitYCbCr); + + // Create textures. We don't allocate any data for the second field at this point + // (just create the texture state with the samplers), since our default assumed + // resolution is progressive. + switch (pixel_format) { + case bmusb::PixelFormat_8BitYCbCr: + glGenTextures(2, userdata[frame_idx].tex_y); + check_error(); + glGenTextures(2, userdata[frame_idx].tex_cbcr); + check_error(); + break; + case bmusb::PixelFormat_10BitYCbCr: + glGenTextures(2, userdata[frame_idx].tex_v210); + check_error(); + glGenTextures(2, userdata[frame_idx].tex_444); + check_error(); + break; + case bmusb::PixelFormat_8BitBGRA: + glGenTextures(2, userdata[frame_idx].tex_rgba); + check_error(); + break; + case bmusb::PixelFormat_8BitYCbCrPlanar: + glGenTextures(2, userdata[frame_idx].tex_y); + check_error(); + glGenTextures(2, userdata[frame_idx].tex_cb); + check_error(); + glGenTextures(2, userdata[frame_idx].tex_cr); + check_error(); + break; + default: + assert(false); + } + + userdata[frame_idx].last_width[0] = width; + userdata[frame_idx].last_height[0] = height; + userdata[frame_idx].last_cbcr_width[0] = width / 2; + userdata[frame_idx].last_cbcr_height[0] = height; + userdata[frame_idx].last_v210_width[0] = 0; + + userdata[frame_idx].last_width[1] = 0; + userdata[frame_idx].last_height[1] = 0; + userdata[frame_idx].last_cbcr_width[1] = 0; + userdata[frame_idx].last_cbcr_height[1] = 0; + userdata[frame_idx].last_v210_width[1] = 0; + + userdata[frame_idx].last_interlaced = false; + userdata[frame_idx].last_has_signal = false; + userdata[frame_idx].last_is_connected = false; + for (unsigned field = 0; field < 2; ++field) { + switch (pixel_format) { + case bmusb::PixelFormat_10BitYCbCr: { + const size_t v210_width = v210Converter::get_minimum_v210_texture_width(width); + + // Seemingly we need to set the minification filter even though + // shader image loads don't use them, or NVIDIA will just give us + // zero back. + glBindTexture(GL_TEXTURE_2D, userdata[frame_idx].tex_v210[field]); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + if (field == 0) { + userdata[frame_idx].last_v210_width[0] = v210_width; + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB10_A2, v210_width, height, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, nullptr); + check_error(); + } + + glBindTexture(GL_TEXTURE_2D, userdata[frame_idx].tex_444[field]); + check_error(); + set_clamp_to_edge(); + if (field == 0) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB10_A2, width, height, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, nullptr); + check_error(); + } + break; + } + case bmusb::PixelFormat_8BitYCbCr: + glBindTexture(GL_TEXTURE_2D, userdata[frame_idx].tex_y[field]); + check_error(); + set_clamp_to_edge(); + if (field == 0) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + } + + glBindTexture(GL_TEXTURE_2D, userdata[frame_idx].tex_cbcr[field]); + check_error(); + set_clamp_to_edge(); + if (field == 0) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, width / 2, height, 0, GL_RG, GL_UNSIGNED_BYTE, nullptr); + check_error(); + } + break; + case bmusb::PixelFormat_8BitBGRA: + glBindTexture(GL_TEXTURE_2D, userdata[frame_idx].tex_rgba[field]); + check_error(); + set_clamp_to_edge(); + if (field == 0) { + if (global_flags.can_disable_srgb_decoder) { // See the comments in tweaked_inputs.h. + glTexImage2D(GL_TEXTURE_2D, 0, GL_SRGB8_ALPHA8, width, height, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, nullptr); + } else { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, nullptr); + } + check_error(); + } + break; + case bmusb::PixelFormat_8BitYCbCrPlanar: + glBindTexture(GL_TEXTURE_2D, userdata[frame_idx].tex_y[field]); + check_error(); + set_clamp_to_edge(); + if (field == 0) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + } + + glBindTexture(GL_TEXTURE_2D, userdata[frame_idx].tex_cb[field]); + check_error(); + set_clamp_to_edge(); + if (field == 0) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width / 2, height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + } + + glBindTexture(GL_TEXTURE_2D, userdata[frame_idx].tex_cr[field]); + check_error(); + set_clamp_to_edge(); + if (field == 0) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width / 2, height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + } + break; + default: + assert(false); + } + } + + freelist.push(frame); +} + +PBOFrameAllocator::~PBOFrameAllocator() +{ + while (!freelist.empty()) { + Frame frame = freelist.front(); + freelist.pop(); + destroy_frame(&frame); + } +} + +void PBOFrameAllocator::destroy_frame(Frame *frame) +{ + GLuint pbo = ((Userdata *)frame->userdata)->pbo; + glBindBuffer(buffer, pbo); + check_error(); + glUnmapBuffer(buffer); + check_error(); + glBindBuffer(buffer, 0); + check_error(); + glDeleteBuffers(1, &pbo); + check_error(); + switch (pixel_format) { + case bmusb::PixelFormat_10BitYCbCr: + glDeleteTextures(2, ((Userdata *)frame->userdata)->tex_v210); + check_error(); + glDeleteTextures(2, ((Userdata *)frame->userdata)->tex_444); + check_error(); + break; + case bmusb::PixelFormat_8BitYCbCr: + glDeleteTextures(2, ((Userdata *)frame->userdata)->tex_y); + check_error(); + glDeleteTextures(2, ((Userdata *)frame->userdata)->tex_cbcr); + check_error(); + break; + case bmusb::PixelFormat_8BitBGRA: + glDeleteTextures(2, ((Userdata *)frame->userdata)->tex_rgba); + check_error(); + break; + case bmusb::PixelFormat_8BitYCbCrPlanar: + glDeleteTextures(2, ((Userdata *)frame->userdata)->tex_y); + check_error(); + glDeleteTextures(2, ((Userdata *)frame->userdata)->tex_cb); + check_error(); + glDeleteTextures(2, ((Userdata *)frame->userdata)->tex_cr); + check_error(); + break; + default: + assert(false); + } +} +//static int sumsum = 0; + +bmusb::FrameAllocator::Frame PBOFrameAllocator::alloc_frame() +{ + Frame vf; + + unique_lock lock(freelist_mutex); // Meh. + if (freelist.empty()) { + printf("Frame overrun (no more spare PBO frames), dropping frame!\n"); + } else { + //fprintf(stderr, "freelist has %d allocated\n", ++sumsum); + vf = freelist.front(); + freelist.pop(); // Meh. + } + vf.len = 0; + vf.overflow = 0; + return vf; +} + +void PBOFrameAllocator::release_frame(Frame frame) +{ + if (frame.overflow > 0) { + printf("%d bytes overflow after last (PBO) frame\n", int(frame.overflow)); + } + +#if 0 + // Poison the page. (Note that this might be bogus if you don't have an OpenGL context.) + memset(frame.data, 0, frame.size); + Userdata *userdata = (Userdata *)frame.userdata; + for (unsigned field = 0; field < 2; ++field) { + glBindTexture(GL_TEXTURE_2D, userdata->tex_y[field]); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, userdata->last_width[field], userdata->last_height[field], 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + + glBindTexture(GL_TEXTURE_2D, userdata->tex_cbcr[field]); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, userdata->last_width[field] / 2, userdata->last_height[field], 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + check_error(); + } +#endif + + unique_lock lock(freelist_mutex); + freelist.push(frame); + //--sumsum; +} diff --git a/nageru/pbo_frame_allocator.h b/nageru/pbo_frame_allocator.h new file mode 100644 index 0000000..eead7f4 --- /dev/null +++ b/nageru/pbo_frame_allocator.h @@ -0,0 +1,68 @@ +#ifndef _PBO_FRAME_ALLOCATOR +#define _PBO_FRAME_ALLOCATOR 1 + +#include +#include +#include +#include +#include +#include + +#include + +#include "bmusb/bmusb.h" + +// An allocator that allocates straight into OpenGL pinned memory. +// Meant for video frames only. We use a queue rather than a stack, +// since we want to maximize pipelineability. +class PBOFrameAllocator : public bmusb::FrameAllocator { +public: + // Note: You need to have an OpenGL context when calling + // the constructor. + PBOFrameAllocator(bmusb::PixelFormat pixel_format, + size_t frame_size, + GLuint width, GLuint height, + size_t num_queued_frames = 16, + GLenum buffer = GL_PIXEL_UNPACK_BUFFER_ARB, + GLenum permissions = GL_MAP_WRITE_BIT, + GLenum map_bits = GL_MAP_FLUSH_EXPLICIT_BIT); + ~PBOFrameAllocator() override; + Frame alloc_frame() override; + void release_frame(Frame frame) override; + + struct Userdata { + GLuint pbo; + + // NOTE: These frames typically go into LiveInputWrapper, which is + // configured to accept one type of frame only. In other words, + // the existence of a format field doesn't mean you can set it + // freely at runtime. + bmusb::PixelFormat pixel_format; + + // Used only for PixelFormat_8BitYCbCrPlanar. + movit::YCbCrFormat ycbcr_format; + + // The second set is only used for the second field of interlaced inputs. + GLuint tex_y[2], tex_cbcr[2]; // For PixelFormat_8BitYCbCr. + GLuint tex_cb[2], tex_cr[2]; // For PixelFormat_8BitYCbCrPlanar (which also uses tex_y). + GLuint tex_v210[2], tex_444[2]; // For PixelFormat_10BitYCbCr. + GLuint tex_rgba[2]; // For PixelFormat_8BitBGRA. + GLuint last_width[2], last_height[2]; + GLuint last_cbcr_width[2], last_cbcr_height[2]; + GLuint last_v210_width[2]; // PixelFormat_10BitYCbCr. + bool last_interlaced, last_has_signal, last_is_connected; + unsigned last_frame_rate_nom, last_frame_rate_den; + }; + +private: + void init_frame(size_t frame_idx, size_t frame_size, GLuint width, GLuint height, GLenum permissions, GLenum map_bits); + void destroy_frame(Frame *frame); + + bmusb::PixelFormat pixel_format; + std::mutex freelist_mutex; + std::queue freelist; + GLenum buffer; + std::unique_ptr userdata; +}; + +#endif // !defined(_PBO_FRAME_ALLOCATOR) diff --git a/nageru/piecewise_interpolator.cpp b/nageru/piecewise_interpolator.cpp new file mode 100644 index 0000000..3f07247 --- /dev/null +++ b/nageru/piecewise_interpolator.cpp @@ -0,0 +1,46 @@ +#include "piecewise_interpolator.h" + +#include + +double PiecewiseInterpolator::fraction_to_db(double db) const +{ + if (db >= control_points[0].fraction) { + return control_points[0].db_value; + } + if (db <= control_points.back().fraction) { + return control_points.back().db_value; + } + for (unsigned i = 1; i < control_points.size(); ++i) { + const double x0 = control_points[i].fraction; + const double x1 = control_points[i - 1].fraction; + const double y0 = control_points[i].db_value; + const double y1 = control_points[i - 1].db_value; + if (db >= x0 && db <= x1) { + const double t = (db - x0) / (x1 - x0); + return y0 + t * (y1 - y0); + } + } + assert(false); +} + +double PiecewiseInterpolator::db_to_fraction(double x) const +{ + if (x >= control_points[0].db_value) { + return control_points[0].fraction; + } + if (x <= control_points.back().db_value) { + return control_points.back().fraction; + } + for (unsigned i = 1; i < control_points.size(); ++i) { + const double x0 = control_points[i].db_value; + const double x1 = control_points[i - 1].db_value; + const double y0 = control_points[i].fraction; + const double y1 = control_points[i - 1].fraction; + if (x >= x0 && x <= x1) { + const double t = (x - x0) / (x1 - x0); + return y0 + t * (y1 - y0); + } + } + assert(false); +} + diff --git a/nageru/piecewise_interpolator.h b/nageru/piecewise_interpolator.h new file mode 100644 index 0000000..17a9d8b --- /dev/null +++ b/nageru/piecewise_interpolator.h @@ -0,0 +1,27 @@ +#ifndef _PIECEWISE_INTERPOLATOR_H +#define _PIECEWISE_INTERPOLATOR_H + +// A class to do piecewise linear interpolation of one scale to another +// (and back). Typically used to implement nonlinear dB mappings for sliders +// or meters, thus the nomenclature. + +#include + +class PiecewiseInterpolator { +public: + // Both dB and fraction values must go from high to low. + struct ControlPoint { + double db_value; + double fraction; + }; + PiecewiseInterpolator(const std::vector &control_points) + : control_points(control_points) {} + + double fraction_to_db(double db) const; + double db_to_fraction(double x) const; + +private: + const std::vector control_points; +}; + +#endif // !defined(_PIECEWISE_INTERPOLATOR_H) diff --git a/nageru/post_to_main_thread.h b/nageru/post_to_main_thread.h new file mode 100644 index 0000000..0462c7b --- /dev/null +++ b/nageru/post_to_main_thread.h @@ -0,0 +1,16 @@ +#ifndef _POST_TO_MAIN_THREAD_H +#define _POST_TO_MAIN_THREAD_H 1 + +#include +#include +#include + +// http://stackoverflow.com/questions/21646467/how-to-execute-a-functor-in-a-given-thread-in-qt-gcd-style +template +static inline void post_to_main_thread(F &&fun) +{ + QObject signalSource; + QObject::connect(&signalSource, &QObject::destroyed, qApp, std::move(fun)); +} + +#endif // !defined(_POST_TO_MAIN_THREAD_H) diff --git a/nageru/print_latency.cpp b/nageru/print_latency.cpp new file mode 100644 index 0000000..72440ae --- /dev/null +++ b/nageru/print_latency.cpp @@ -0,0 +1,131 @@ +#include "print_latency.h" + +#include "flags.h" +#include "metrics.h" +#include "mixer.h" + +#include +#include +#include +#include + +using namespace std; +using namespace std::chrono; + +ReceivedTimestamps find_received_timestamp(const vector &input_frames) +{ + unsigned num_cards = global_mixer->get_num_cards(); + assert(input_frames.size() == num_cards * FRAME_HISTORY_LENGTH); + + ReceivedTimestamps ts; + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + for (unsigned frame_index = 0; frame_index < FRAME_HISTORY_LENGTH; ++frame_index) { + const RefCountedFrame &input_frame = input_frames[card_index * FRAME_HISTORY_LENGTH + frame_index]; + if (input_frame == nullptr || + (frame_index > 0 && input_frame.get() == input_frames[card_index * FRAME_HISTORY_LENGTH + frame_index - 1].get())) { + ts.ts.push_back(steady_clock::time_point::min()); + } else { + ts.ts.push_back(input_frame->received_timestamp); + } + } + } + return ts; +} + +void LatencyHistogram::init(const string &measuring_point) +{ + unsigned num_cards = global_flags.num_cards; // The mixer might not be ready yet. + summaries.resize(num_cards * FRAME_HISTORY_LENGTH * 2); + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + char card_index_str[64]; + snprintf(card_index_str, sizeof(card_index_str), "%u", card_index); + summaries[card_index].resize(FRAME_HISTORY_LENGTH); + for (unsigned frame_index = 0; frame_index < FRAME_HISTORY_LENGTH; ++frame_index) { + char frame_index_str[64]; + snprintf(frame_index_str, sizeof(frame_index_str), "%u", frame_index); + + vector quantiles{0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99}; + summaries[card_index][frame_index].reset(new Summary[3]); + summaries[card_index][frame_index][0].init(quantiles, 60.0); + summaries[card_index][frame_index][1].init(quantiles, 60.0); + summaries[card_index][frame_index][2].init(quantiles, 60.0); + global_metrics.add("latency_seconds", + {{ "measuring_point", measuring_point }, + { "card", card_index_str }, + { "frame_age", frame_index_str }, + { "frame_type", "i/p" }}, + &summaries[card_index][frame_index][0], + (frame_index == 0) ? Metrics::PRINT_ALWAYS : Metrics::PRINT_WHEN_NONEMPTY); + global_metrics.add("latency_seconds", + {{ "measuring_point", measuring_point }, + { "card", card_index_str }, + { "frame_age", frame_index_str }, + { "frame_type", "b" }}, + &summaries[card_index][frame_index][1], + Metrics::PRINT_WHEN_NONEMPTY); + global_metrics.add("latency_seconds", + {{ "measuring_point", measuring_point }, + { "card", card_index_str }, + { "frame_age", frame_index_str }, + { "frame_type", "total" }}, + &summaries[card_index][frame_index][2], + (frame_index == 0) ? Metrics::PRINT_ALWAYS : Metrics::PRINT_WHEN_NONEMPTY); + } + } +} + +void print_latency(const char *header, const ReceivedTimestamps &received_ts, bool is_b_frame, int *frameno, LatencyHistogram *histogram) +{ + if (received_ts.ts.empty()) + return; + + const steady_clock::time_point now = steady_clock::now(); + + if (global_mixer == nullptr) { + // Kaeru. + assert(received_ts.ts.size() == 1); + steady_clock::time_point ts = received_ts.ts[0]; + if (ts != steady_clock::time_point::min()) { + duration latency = now - ts; + histogram->summaries[0][0][is_b_frame].count_event(latency.count()); + histogram->summaries[0][0][2].count_event(latency.count()); + } + } else { + unsigned num_cards = global_mixer->get_num_cards(); + assert(received_ts.ts.size() == num_cards * FRAME_HISTORY_LENGTH); + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + for (unsigned frame_index = 0; frame_index < FRAME_HISTORY_LENGTH; ++frame_index) { + steady_clock::time_point ts = received_ts.ts[card_index * FRAME_HISTORY_LENGTH + frame_index]; + if (ts == steady_clock::time_point::min()) { + continue; + } + duration latency = now - ts; + histogram->summaries[card_index][frame_index][is_b_frame].count_event(latency.count()); + histogram->summaries[card_index][frame_index][2].count_event(latency.count()); + } + } + } + + // 101 is chosen so that it's prime, which is unlikely to get the same frame type every time. + if (global_flags.print_video_latency && (++*frameno % 101) == 0) { + // Find min and max timestamp of all input frames that have a timestamp. + steady_clock::time_point min_ts = steady_clock::time_point::max(), max_ts = steady_clock::time_point::min(); + for (const auto &ts : received_ts.ts) { + if (ts > steady_clock::time_point::min()) { + min_ts = min(min_ts, ts); + max_ts = max(max_ts, ts); + } + } + duration lowest_latency = now - max_ts; + duration highest_latency = now - min_ts; + + printf("%-60s %4.0f ms (lowest-latency input), %4.0f ms (highest-latency input)", + header, 1e3 * lowest_latency.count(), 1e3 * highest_latency.count()); + + if (is_b_frame) { + printf(" [on B-frame; potential extra latency]\n"); + } else { + printf("\n"); + } + } +} diff --git a/nageru/print_latency.h b/nageru/print_latency.h new file mode 100644 index 0000000..d80ac88 --- /dev/null +++ b/nageru/print_latency.h @@ -0,0 +1,32 @@ +#ifndef _PRINT_LATENCY_H +#define _PRINT_LATENCY_H 1 + +// A small utility function to print the latency between two end points +// (typically when the frame was received from the video card, and some +// point when the frame is ready to be output in some form). + +#include +#include +#include + +#include "ref_counted_frame.h" +#include "metrics.h" + +// Since every output frame is based on multiple input frames, we need +// more than one start timestamp; one for each input. +// For all of these, steady_clock::time_point::min() is used for “not set”. +struct ReceivedTimestamps { + std::vector ts; +}; +struct LatencyHistogram { + void init(const std::string &measuring_point); // Initializes histograms and registers them in global_metrics. + + // Indices: card number, frame history number, b-frame or not (1/0, where 2 counts both). + std::vector>> summaries; +}; + +ReceivedTimestamps find_received_timestamp(const std::vector &input_frames); + +void print_latency(const char *header, const ReceivedTimestamps &received_ts, bool is_b_frame, int *frameno, LatencyHistogram *histogram); + +#endif // !defined(_PRINT_LATENCY_H) diff --git a/nageru/quicksync_encoder.cpp b/nageru/quicksync_encoder.cpp new file mode 100644 index 0000000..5200ce6 --- /dev/null +++ b/nageru/quicksync_encoder.cpp @@ -0,0 +1,2183 @@ +#include "quicksync_encoder.h" + +#include +#include // Must be above the Xlib includes. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { + +#include +#include +#include +#include + +} // namespace + +#include "audio_encoder.h" +#include "context.h" +#include "defs.h" +#include "disk_space_estimator.h" +#include "ffmpeg_raii.h" +#include "flags.h" +#include "mux.h" +#include "print_latency.h" +#include "quicksync_encoder_impl.h" +#include "ref_counted_frame.h" +#include "timebase.h" +#include "x264_encoder.h" + +using namespace movit; +using namespace std; +using namespace std::chrono; +using namespace std::placeholders; + +class QOpenGLContext; +class QSurface; + +namespace { + +// These need to survive several QuickSyncEncoderImpl instances, +// so they are outside. +once_flag quick_sync_metrics_inited; +LatencyHistogram mixer_latency_histogram, qs_latency_histogram; +MuxMetrics current_file_mux_metrics, total_mux_metrics; +std::atomic metric_current_file_start_time_seconds{0.0 / 0.0}; +std::atomic metric_quick_sync_stalled_frames{0}; + +} // namespace + +#define CHECK_VASTATUS(va_status, func) \ + if (va_status != VA_STATUS_SUCCESS) { \ + fprintf(stderr, "%s:%d (%s) failed with %d\n", __func__, __LINE__, func, va_status); \ + exit(1); \ + } + +#undef BUFFER_OFFSET +#define BUFFER_OFFSET(i) ((char *)NULL + (i)) + +//#include "loadsurface.h" + +#define NAL_REF_IDC_NONE 0 +#define NAL_REF_IDC_LOW 1 +#define NAL_REF_IDC_MEDIUM 2 +#define NAL_REF_IDC_HIGH 3 + +#define NAL_NON_IDR 1 +#define NAL_IDR 5 +#define NAL_SPS 7 +#define NAL_PPS 8 +#define NAL_SEI 6 + +#define SLICE_TYPE_P 0 +#define SLICE_TYPE_B 1 +#define SLICE_TYPE_I 2 +#define IS_P_SLICE(type) (SLICE_TYPE_P == (type)) +#define IS_B_SLICE(type) (SLICE_TYPE_B == (type)) +#define IS_I_SLICE(type) (SLICE_TYPE_I == (type)) + + +#define ENTROPY_MODE_CAVLC 0 +#define ENTROPY_MODE_CABAC 1 + +#define PROFILE_IDC_BASELINE 66 +#define PROFILE_IDC_MAIN 77 +#define PROFILE_IDC_HIGH 100 + +#define BITSTREAM_ALLOCATE_STEPPING 4096 + +static constexpr unsigned int MaxFrameNum = (2<<16); +static constexpr unsigned int MaxPicOrderCntLsb = (2<<8); +static constexpr unsigned int Log2MaxFrameNum = 16; +static constexpr unsigned int Log2MaxPicOrderCntLsb = 8; + +using namespace std; + +// Supposedly vaRenderPicture() is supposed to destroy the buffer implicitly, +// but if we don't delete it here, we get leaks. The GStreamer implementation +// does the same. +static void render_picture_and_delete(VADisplay dpy, VAContextID context, VABufferID *buffers, int num_buffers) +{ + VAStatus va_status = vaRenderPicture(dpy, context, buffers, num_buffers); + CHECK_VASTATUS(va_status, "vaRenderPicture"); + + for (int i = 0; i < num_buffers; ++i) { + va_status = vaDestroyBuffer(dpy, buffers[i]); + CHECK_VASTATUS(va_status, "vaDestroyBuffer"); + } +} + +static unsigned int +va_swap32(unsigned int val) +{ + unsigned char *pval = (unsigned char *)&val; + + return ((pval[0] << 24) | + (pval[1] << 16) | + (pval[2] << 8) | + (pval[3] << 0)); +} + +static void +bitstream_start(bitstream *bs) +{ + bs->max_size_in_dword = BITSTREAM_ALLOCATE_STEPPING; + bs->buffer = (unsigned int *)calloc(bs->max_size_in_dword * sizeof(int), 1); + bs->bit_offset = 0; +} + +static void +bitstream_end(bitstream *bs) +{ + int pos = (bs->bit_offset >> 5); + int bit_offset = (bs->bit_offset & 0x1f); + int bit_left = 32 - bit_offset; + + if (bit_offset) { + bs->buffer[pos] = va_swap32((bs->buffer[pos] << bit_left)); + } +} + +static void +bitstream_put_ui(bitstream *bs, unsigned int val, int size_in_bits) +{ + int pos = (bs->bit_offset >> 5); + int bit_offset = (bs->bit_offset & 0x1f); + int bit_left = 32 - bit_offset; + + if (!size_in_bits) + return; + + bs->bit_offset += size_in_bits; + + if (bit_left > size_in_bits) { + bs->buffer[pos] = (bs->buffer[pos] << size_in_bits | val); + } else { + size_in_bits -= bit_left; + if (bit_left >= 32) { + bs->buffer[pos] = (val >> size_in_bits); + } else { + bs->buffer[pos] = (bs->buffer[pos] << bit_left) | (val >> size_in_bits); + } + bs->buffer[pos] = va_swap32(bs->buffer[pos]); + + if (pos + 1 == bs->max_size_in_dword) { + bs->max_size_in_dword += BITSTREAM_ALLOCATE_STEPPING; + bs->buffer = (unsigned int *)realloc(bs->buffer, bs->max_size_in_dword * sizeof(unsigned int)); + } + + bs->buffer[pos + 1] = val; + } +} + +static void +bitstream_put_ue(bitstream *bs, unsigned int val) +{ + int size_in_bits = 0; + int tmp_val = ++val; + + while (tmp_val) { + tmp_val >>= 1; + size_in_bits++; + } + + bitstream_put_ui(bs, 0, size_in_bits - 1); // leading zero + bitstream_put_ui(bs, val, size_in_bits); +} + +static void +bitstream_put_se(bitstream *bs, int val) +{ + unsigned int new_val; + + if (val <= 0) + new_val = -2 * val; + else + new_val = 2 * val - 1; + + bitstream_put_ue(bs, new_val); +} + +static void +bitstream_byte_aligning(bitstream *bs, int bit) +{ + int bit_offset = (bs->bit_offset & 0x7); + int bit_left = 8 - bit_offset; + int new_val; + + if (!bit_offset) + return; + + assert(bit == 0 || bit == 1); + + if (bit) + new_val = (1 << bit_left) - 1; + else + new_val = 0; + + bitstream_put_ui(bs, new_val, bit_left); +} + +static void +rbsp_trailing_bits(bitstream *bs) +{ + bitstream_put_ui(bs, 1, 1); + bitstream_byte_aligning(bs, 0); +} + +static void nal_start_code_prefix(bitstream *bs) +{ + bitstream_put_ui(bs, 0x00000001, 32); +} + +static void nal_header(bitstream *bs, int nal_ref_idc, int nal_unit_type) +{ + bitstream_put_ui(bs, 0, 1); /* forbidden_zero_bit: 0 */ + bitstream_put_ui(bs, nal_ref_idc, 2); + bitstream_put_ui(bs, nal_unit_type, 5); +} + +void QuickSyncEncoderImpl::sps_rbsp(YCbCrLumaCoefficients ycbcr_coefficients, bitstream *bs) +{ + int profile_idc = PROFILE_IDC_BASELINE; + + if (h264_profile == VAProfileH264High) + profile_idc = PROFILE_IDC_HIGH; + else if (h264_profile == VAProfileH264Main) + profile_idc = PROFILE_IDC_MAIN; + + bitstream_put_ui(bs, profile_idc, 8); /* profile_idc */ + bitstream_put_ui(bs, !!(constraint_set_flag & 1), 1); /* constraint_set0_flag */ + bitstream_put_ui(bs, !!(constraint_set_flag & 2), 1); /* constraint_set1_flag */ + bitstream_put_ui(bs, !!(constraint_set_flag & 4), 1); /* constraint_set2_flag */ + bitstream_put_ui(bs, !!(constraint_set_flag & 8), 1); /* constraint_set3_flag */ + bitstream_put_ui(bs, 0, 4); /* reserved_zero_4bits */ + bitstream_put_ui(bs, seq_param.level_idc, 8); /* level_idc */ + bitstream_put_ue(bs, seq_param.seq_parameter_set_id); /* seq_parameter_set_id */ + + if ( profile_idc == PROFILE_IDC_HIGH) { + bitstream_put_ue(bs, 1); /* chroma_format_idc = 1, 4:2:0 */ + bitstream_put_ue(bs, 0); /* bit_depth_luma_minus8 */ + bitstream_put_ue(bs, 0); /* bit_depth_chroma_minus8 */ + bitstream_put_ui(bs, 0, 1); /* qpprime_y_zero_transform_bypass_flag */ + bitstream_put_ui(bs, 0, 1); /* seq_scaling_matrix_present_flag */ + } + + bitstream_put_ue(bs, seq_param.seq_fields.bits.log2_max_frame_num_minus4); /* log2_max_frame_num_minus4 */ + bitstream_put_ue(bs, seq_param.seq_fields.bits.pic_order_cnt_type); /* pic_order_cnt_type */ + + if (seq_param.seq_fields.bits.pic_order_cnt_type == 0) + bitstream_put_ue(bs, seq_param.seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4); /* log2_max_pic_order_cnt_lsb_minus4 */ + else { + assert(0); + } + + bitstream_put_ue(bs, seq_param.max_num_ref_frames); /* num_ref_frames */ + bitstream_put_ui(bs, 0, 1); /* gaps_in_frame_num_value_allowed_flag */ + + bitstream_put_ue(bs, seq_param.picture_width_in_mbs - 1); /* pic_width_in_mbs_minus1 */ + bitstream_put_ue(bs, seq_param.picture_height_in_mbs - 1); /* pic_height_in_map_units_minus1 */ + bitstream_put_ui(bs, seq_param.seq_fields.bits.frame_mbs_only_flag, 1); /* frame_mbs_only_flag */ + + if (!seq_param.seq_fields.bits.frame_mbs_only_flag) { + assert(0); + } + + bitstream_put_ui(bs, seq_param.seq_fields.bits.direct_8x8_inference_flag, 1); /* direct_8x8_inference_flag */ + bitstream_put_ui(bs, seq_param.frame_cropping_flag, 1); /* frame_cropping_flag */ + + if (seq_param.frame_cropping_flag) { + bitstream_put_ue(bs, seq_param.frame_crop_left_offset); /* frame_crop_left_offset */ + bitstream_put_ue(bs, seq_param.frame_crop_right_offset); /* frame_crop_right_offset */ + bitstream_put_ue(bs, seq_param.frame_crop_top_offset); /* frame_crop_top_offset */ + bitstream_put_ue(bs, seq_param.frame_crop_bottom_offset); /* frame_crop_bottom_offset */ + } + + //if ( frame_bit_rate < 0 ) { //TODO EW: the vui header isn't correct + if ( false ) { + bitstream_put_ui(bs, 0, 1); /* vui_parameters_present_flag */ + } else { + // See H.264 annex E for the definition of this header. + bitstream_put_ui(bs, 1, 1); /* vui_parameters_present_flag */ + bitstream_put_ui(bs, 0, 1); /* aspect_ratio_info_present_flag */ + bitstream_put_ui(bs, 0, 1); /* overscan_info_present_flag */ + bitstream_put_ui(bs, 1, 1); /* video_signal_type_present_flag */ + { + bitstream_put_ui(bs, 5, 3); /* video_format (5 = Unspecified) */ + bitstream_put_ui(bs, 0, 1); /* video_full_range_flag */ + bitstream_put_ui(bs, 1, 1); /* colour_description_present_flag */ + { + bitstream_put_ui(bs, 1, 8); /* colour_primaries (1 = BT.709) */ + bitstream_put_ui(bs, 13, 8); /* transfer_characteristics (13 = sRGB) */ + if (ycbcr_coefficients == YCBCR_REC_709) { + bitstream_put_ui(bs, 1, 8); /* matrix_coefficients (1 = BT.709) */ + } else { + assert(ycbcr_coefficients == YCBCR_REC_601); + bitstream_put_ui(bs, 6, 8); /* matrix_coefficients (6 = BT.601/SMPTE 170M) */ + } + } + } + bitstream_put_ui(bs, 0, 1); /* chroma_loc_info_present_flag */ + bitstream_put_ui(bs, 1, 1); /* timing_info_present_flag */ + { + bitstream_put_ui(bs, 1, 32); // FPS + bitstream_put_ui(bs, TIMEBASE * 2, 32); // FPS + bitstream_put_ui(bs, 1, 1); + } + bitstream_put_ui(bs, 1, 1); /* nal_hrd_parameters_present_flag */ + { + // hrd_parameters + bitstream_put_ue(bs, 0); /* cpb_cnt_minus1 */ + bitstream_put_ui(bs, 4, 4); /* bit_rate_scale */ + bitstream_put_ui(bs, 6, 4); /* cpb_size_scale */ + + bitstream_put_ue(bs, frame_bitrate - 1); /* bit_rate_value_minus1[0] */ + bitstream_put_ue(bs, frame_bitrate*8 - 1); /* cpb_size_value_minus1[0] */ + bitstream_put_ui(bs, 1, 1); /* cbr_flag[0] */ + + bitstream_put_ui(bs, 23, 5); /* initial_cpb_removal_delay_length_minus1 */ + bitstream_put_ui(bs, 23, 5); /* cpb_removal_delay_length_minus1 */ + bitstream_put_ui(bs, 23, 5); /* dpb_output_delay_length_minus1 */ + bitstream_put_ui(bs, 23, 5); /* time_offset_length */ + } + bitstream_put_ui(bs, 0, 1); /* vcl_hrd_parameters_present_flag */ + bitstream_put_ui(bs, 0, 1); /* low_delay_hrd_flag */ + + bitstream_put_ui(bs, 0, 1); /* pic_struct_present_flag */ + bitstream_put_ui(bs, 0, 1); /* bitstream_restriction_flag */ + } + + rbsp_trailing_bits(bs); /* rbsp_trailing_bits */ +} + + +void QuickSyncEncoderImpl::pps_rbsp(bitstream *bs) +{ + bitstream_put_ue(bs, pic_param.pic_parameter_set_id); /* pic_parameter_set_id */ + bitstream_put_ue(bs, pic_param.seq_parameter_set_id); /* seq_parameter_set_id */ + + bitstream_put_ui(bs, pic_param.pic_fields.bits.entropy_coding_mode_flag, 1); /* entropy_coding_mode_flag */ + + bitstream_put_ui(bs, 0, 1); /* pic_order_present_flag: 0 */ + + bitstream_put_ue(bs, 0); /* num_slice_groups_minus1 */ + + bitstream_put_ue(bs, pic_param.num_ref_idx_l0_active_minus1); /* num_ref_idx_l0_active_minus1 */ + bitstream_put_ue(bs, pic_param.num_ref_idx_l1_active_minus1); /* num_ref_idx_l1_active_minus1 1 */ + + bitstream_put_ui(bs, pic_param.pic_fields.bits.weighted_pred_flag, 1); /* weighted_pred_flag: 0 */ + bitstream_put_ui(bs, pic_param.pic_fields.bits.weighted_bipred_idc, 2); /* weighted_bipred_idc: 0 */ + + bitstream_put_se(bs, pic_param.pic_init_qp - 26); /* pic_init_qp_minus26 */ + bitstream_put_se(bs, 0); /* pic_init_qs_minus26 */ + bitstream_put_se(bs, 0); /* chroma_qp_index_offset */ + + bitstream_put_ui(bs, pic_param.pic_fields.bits.deblocking_filter_control_present_flag, 1); /* deblocking_filter_control_present_flag */ + bitstream_put_ui(bs, 0, 1); /* constrained_intra_pred_flag */ + bitstream_put_ui(bs, 0, 1); /* redundant_pic_cnt_present_flag */ + + /* more_rbsp_data */ + bitstream_put_ui(bs, pic_param.pic_fields.bits.transform_8x8_mode_flag, 1); /*transform_8x8_mode_flag */ + bitstream_put_ui(bs, 0, 1); /* pic_scaling_matrix_present_flag */ + bitstream_put_se(bs, pic_param.second_chroma_qp_index_offset ); /*second_chroma_qp_index_offset */ + + rbsp_trailing_bits(bs); +} + +void QuickSyncEncoderImpl::slice_header(bitstream *bs) +{ + int first_mb_in_slice = slice_param.macroblock_address; + + bitstream_put_ue(bs, first_mb_in_slice); /* first_mb_in_slice: 0 */ + bitstream_put_ue(bs, slice_param.slice_type); /* slice_type */ + bitstream_put_ue(bs, slice_param.pic_parameter_set_id); /* pic_parameter_set_id: 0 */ + bitstream_put_ui(bs, pic_param.frame_num, seq_param.seq_fields.bits.log2_max_frame_num_minus4 + 4); /* frame_num */ + + /* frame_mbs_only_flag == 1 */ + if (!seq_param.seq_fields.bits.frame_mbs_only_flag) { + /* FIXME: */ + assert(0); + } + + if (pic_param.pic_fields.bits.idr_pic_flag) + bitstream_put_ue(bs, slice_param.idr_pic_id); /* idr_pic_id: 0 */ + + if (seq_param.seq_fields.bits.pic_order_cnt_type == 0) { + bitstream_put_ui(bs, pic_param.CurrPic.TopFieldOrderCnt, seq_param.seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4 + 4); + /* pic_order_present_flag == 0 */ + } else { + /* FIXME: */ + assert(0); + } + + /* redundant_pic_cnt_present_flag == 0 */ + /* slice type */ + if (IS_P_SLICE(slice_param.slice_type)) { + bitstream_put_ui(bs, slice_param.num_ref_idx_active_override_flag, 1); /* num_ref_idx_active_override_flag: */ + + if (slice_param.num_ref_idx_active_override_flag) + bitstream_put_ue(bs, slice_param.num_ref_idx_l0_active_minus1); + + /* ref_pic_list_reordering */ + bitstream_put_ui(bs, 0, 1); /* ref_pic_list_reordering_flag_l0: 0 */ + } else if (IS_B_SLICE(slice_param.slice_type)) { + bitstream_put_ui(bs, slice_param.direct_spatial_mv_pred_flag, 1); /* direct_spatial_mv_pred: 1 */ + + bitstream_put_ui(bs, slice_param.num_ref_idx_active_override_flag, 1); /* num_ref_idx_active_override_flag: */ + + if (slice_param.num_ref_idx_active_override_flag) { + bitstream_put_ue(bs, slice_param.num_ref_idx_l0_active_minus1); + bitstream_put_ue(bs, slice_param.num_ref_idx_l1_active_minus1); + } + + /* ref_pic_list_reordering */ + bitstream_put_ui(bs, 0, 1); /* ref_pic_list_reordering_flag_l0: 0 */ + bitstream_put_ui(bs, 0, 1); /* ref_pic_list_reordering_flag_l1: 0 */ + } + + if ((pic_param.pic_fields.bits.weighted_pred_flag && + IS_P_SLICE(slice_param.slice_type)) || + ((pic_param.pic_fields.bits.weighted_bipred_idc == 1) && + IS_B_SLICE(slice_param.slice_type))) { + /* FIXME: fill weight/offset table */ + assert(0); + } + + /* dec_ref_pic_marking */ + if (pic_param.pic_fields.bits.reference_pic_flag) { /* nal_ref_idc != 0 */ + unsigned char no_output_of_prior_pics_flag = 0; + unsigned char long_term_reference_flag = 0; + unsigned char adaptive_ref_pic_marking_mode_flag = 0; + + if (pic_param.pic_fields.bits.idr_pic_flag) { + bitstream_put_ui(bs, no_output_of_prior_pics_flag, 1); /* no_output_of_prior_pics_flag: 0 */ + bitstream_put_ui(bs, long_term_reference_flag, 1); /* long_term_reference_flag: 0 */ + } else { + bitstream_put_ui(bs, adaptive_ref_pic_marking_mode_flag, 1); /* adaptive_ref_pic_marking_mode_flag: 0 */ + } + } + + if (pic_param.pic_fields.bits.entropy_coding_mode_flag && + !IS_I_SLICE(slice_param.slice_type)) + bitstream_put_ue(bs, slice_param.cabac_init_idc); /* cabac_init_idc: 0 */ + + bitstream_put_se(bs, slice_param.slice_qp_delta); /* slice_qp_delta: 0 */ + + /* ignore for SP/SI */ + + if (pic_param.pic_fields.bits.deblocking_filter_control_present_flag) { + bitstream_put_ue(bs, slice_param.disable_deblocking_filter_idc); /* disable_deblocking_filter_idc: 0 */ + + if (slice_param.disable_deblocking_filter_idc != 1) { + bitstream_put_se(bs, slice_param.slice_alpha_c0_offset_div2); /* slice_alpha_c0_offset_div2: 2 */ + bitstream_put_se(bs, slice_param.slice_beta_offset_div2); /* slice_beta_offset_div2: 2 */ + } + } + + if (pic_param.pic_fields.bits.entropy_coding_mode_flag) { + bitstream_byte_aligning(bs, 1); + } +} + +int QuickSyncEncoderImpl::build_packed_pic_buffer(unsigned char **header_buffer) +{ + bitstream bs; + + bitstream_start(&bs); + nal_start_code_prefix(&bs); + nal_header(&bs, NAL_REF_IDC_HIGH, NAL_PPS); + pps_rbsp(&bs); + bitstream_end(&bs); + + *header_buffer = (unsigned char *)bs.buffer; + return bs.bit_offset; +} + +int +QuickSyncEncoderImpl::build_packed_seq_buffer(YCbCrLumaCoefficients ycbcr_coefficients, unsigned char **header_buffer) +{ + bitstream bs; + + bitstream_start(&bs); + nal_start_code_prefix(&bs); + nal_header(&bs, NAL_REF_IDC_HIGH, NAL_SPS); + sps_rbsp(ycbcr_coefficients, &bs); + bitstream_end(&bs); + + *header_buffer = (unsigned char *)bs.buffer; + return bs.bit_offset; +} + +int QuickSyncEncoderImpl::build_packed_slice_buffer(unsigned char **header_buffer) +{ + bitstream bs; + int is_idr = !!pic_param.pic_fields.bits.idr_pic_flag; + int is_ref = !!pic_param.pic_fields.bits.reference_pic_flag; + + bitstream_start(&bs); + nal_start_code_prefix(&bs); + + if (IS_I_SLICE(slice_param.slice_type)) { + nal_header(&bs, NAL_REF_IDC_HIGH, is_idr ? NAL_IDR : NAL_NON_IDR); + } else if (IS_P_SLICE(slice_param.slice_type)) { + nal_header(&bs, NAL_REF_IDC_MEDIUM, NAL_NON_IDR); + } else { + assert(IS_B_SLICE(slice_param.slice_type)); + nal_header(&bs, is_ref ? NAL_REF_IDC_LOW : NAL_REF_IDC_NONE, NAL_NON_IDR); + } + + slice_header(&bs); + bitstream_end(&bs); + + *header_buffer = (unsigned char *)bs.buffer; + return bs.bit_offset; +} + + +/* + Assume frame sequence is: Frame#0, #1, #2, ..., #M, ..., #X, ... (encoding order) + 1) period between Frame #X and Frame #N = #X - #N + 2) 0 means infinite for intra_period/intra_idr_period, and 0 is invalid for ip_period + 3) intra_idr_period % intra_period (intra_period > 0) and intra_period % ip_period must be 0 + 4) intra_period and intra_idr_period take precedence over ip_period + 5) if ip_period > 1, intra_period and intra_idr_period are not the strict periods + of I/IDR frames, see bellow examples + ------------------------------------------------------------------- + intra_period intra_idr_period ip_period frame sequence (intra_period/intra_idr_period/ip_period) + 0 ignored 1 IDRPPPPPPP ... (No IDR/I any more) + 0 ignored >=2 IDR(PBB)(PBB)... (No IDR/I any more) + 1 0 ignored IDRIIIIIII... (No IDR any more) + 1 1 ignored IDR IDR IDR IDR... + 1 >=2 ignored IDRII IDRII IDR... (1/3/ignore) + >=2 0 1 IDRPPP IPPP I... (3/0/1) + >=2 0 >=2 IDR(PBB)(PBB)(IBB) (6/0/3) + (PBB)(IBB)(PBB)(IBB)... + >=2 >=2 1 IDRPPPPP IPPPPP IPPPPP (6/18/1) + IDRPPPPP IPPPPP IPPPPP... + >=2 >=2 >=2 {IDR(PBB)(PBB)(IBB)(PBB)(IBB)(PBB)} (6/18/3) + {IDR(PBB)(PBB)(IBB)(PBB)(IBB)(PBB)}... + {IDR(PBB)(PBB)(IBB)(PBB)} (6/12/3) + {IDR(PBB)(PBB)(IBB)(PBB)}... + {IDR(PBB)(PBB)} (6/6/3) + {IDR(PBB)(PBB)}. +*/ + +// General pts/dts strategy: +// +// Getting pts and dts right with variable frame rate (VFR) and B-frames can be a +// bit tricky. We assume first of all that the frame rate never goes _above_ +// MAX_FPS, which gives us a frame period N. The decoder can always decode +// in at least this speed, as long at dts <= pts (the frame is not attempted +// presented before it is decoded). Furthermore, we never have longer chains of +// B-frames than a fixed constant C. (In a B-frame chain, we say that the base +// I/P-frame has order O=0, the B-frame depending on it directly has order O=1, +// etc. The last frame in the chain, which no B-frames depend on, is the “tip” +// frame, with an order O <= C.) +// +// Many strategies are possible, but we establish these rules: +// +// - Tip frames have dts = pts - (C-O)*N. +// - Non-tip frames have dts = dts_last + N. +// +// An example, with C=2 and N=10 and the data flow showed with arrows: +// +// I B P B B P +// pts: 30 40 50 60 70 80 +// ↓ ↓ ↓ +// dts: 10 30 20 60 50←40 +// | | ↑ ↑ +// `--|--' | +// `----------' +// +// To show that this works fine also with irregular spacings, let's say that +// the third frame is delayed a bit (something earlier was dropped). Now the +// situation looks like this: +// +// I B P B B P +// pts: 30 40 80 90 100 110 +// ↓ ↓ ↓ +// dts: 10 30 20 90 50←40 +// | | ↑ ↑ +// `--|--' | +// `----------' +// +// The resetting on every tip frame makes sure dts never ends up lagging a lot +// behind pts, and the subtraction of (C-O)*N makes sure pts <= dts. +// +// In the output of this function, if is >= 0, it means to reset the +// dts from the current pts minus , while if it's -1, the frame is not +// a tip frame and should be given a dts based on the previous one. +#define FRAME_P 0 +#define FRAME_B 1 +#define FRAME_I 2 +#define FRAME_IDR 7 +void encoding2display_order( + int encoding_order, int intra_period, + int intra_idr_period, int ip_period, + int *displaying_order, + int *frame_type, int *pts_lag) +{ + int encoding_order_gop = 0; + + *pts_lag = 0; + + if (intra_period == 1) { /* all are I/IDR frames */ + *displaying_order = encoding_order; + if (intra_idr_period == 0) + *frame_type = (encoding_order == 0)?FRAME_IDR:FRAME_I; + else + *frame_type = (encoding_order % intra_idr_period == 0)?FRAME_IDR:FRAME_I; + return; + } + + if (intra_period == 0) + intra_idr_period = 0; + + if (ip_period == 1) { + // No B-frames, sequence is like IDR PPPPP IPPPPP. + encoding_order_gop = (intra_idr_period == 0) ? encoding_order : (encoding_order % intra_idr_period); + *displaying_order = encoding_order; + + if (encoding_order_gop == 0) { /* the first frame */ + *frame_type = FRAME_IDR; + } else if (intra_period != 0 && /* have I frames */ + encoding_order_gop >= 2 && + (encoding_order_gop % intra_period == 0)) { + *frame_type = FRAME_I; + } else { + *frame_type = FRAME_P; + } + return; + } + + // We have B-frames. Sequence is like IDR (PBB)(PBB)(IBB)(PBB). + encoding_order_gop = (intra_idr_period == 0) ? encoding_order : (encoding_order % (intra_idr_period + 1)); + *pts_lag = -1; // Most frames are not tip frames. + + if (encoding_order_gop == 0) { /* the first frame */ + *frame_type = FRAME_IDR; + *displaying_order = encoding_order; + // IDR frames are a special case; I honestly can't find the logic behind + // why this is the right thing, but it seems to line up nicely in practice :-) + *pts_lag = TIMEBASE / MAX_FPS; + } else if (((encoding_order_gop - 1) % ip_period) != 0) { /* B frames */ + *frame_type = FRAME_B; + *displaying_order = encoding_order - 1; + if ((encoding_order_gop % ip_period) == 0) { + *pts_lag = 0; // Last B-frame. + } + } else if (intra_period != 0 && /* have I frames */ + encoding_order_gop >= 2 && + ((encoding_order_gop - 1) / ip_period % (intra_period / ip_period)) == 0) { + *frame_type = FRAME_I; + *displaying_order = encoding_order + ip_period - 1; + } else { + *frame_type = FRAME_P; + *displaying_order = encoding_order + ip_period - 1; + } +} + + +void QuickSyncEncoderImpl::enable_zerocopy_if_possible() +{ + if (global_flags.x264_video_to_disk) { + // Quick Sync is entirely disabled. + use_zerocopy = false; + } else if (global_flags.uncompressed_video_to_http) { + fprintf(stderr, "Disabling zerocopy H.264 encoding due to --http-uncompressed-video.\n"); + use_zerocopy = false; + } else if (global_flags.x264_video_to_http) { + fprintf(stderr, "Disabling zerocopy H.264 encoding due to --http-x264-video.\n"); + use_zerocopy = false; + } else { + use_zerocopy = true; + } + global_flags.use_zerocopy = use_zerocopy; +} + +VADisplayWithCleanup::~VADisplayWithCleanup() +{ + if (va_dpy != nullptr) { + vaTerminate(va_dpy); + } + if (x11_display != nullptr) { + XCloseDisplay(x11_display); + } + if (drm_fd != -1) { + close(drm_fd); + } +} + +unique_ptr va_open_display(const string &va_display) +{ + if (va_display.empty() || va_display[0] != '/') { // An X display. + Display *x11_display = XOpenDisplay(va_display.empty() ? nullptr : va_display.c_str()); + if (x11_display == nullptr) { + fprintf(stderr, "error: can't connect to X server!\n"); + return nullptr; + } + + unique_ptr ret(new VADisplayWithCleanup); + ret->x11_display = x11_display; + ret->can_use_zerocopy = true; + ret->va_dpy = vaGetDisplay(x11_display); + if (ret->va_dpy == nullptr) { + return nullptr; + } + return ret; + } else { // A DRM node on the filesystem (e.g. /dev/dri/renderD128). + int drm_fd = open(va_display.c_str(), O_RDWR); + if (drm_fd == -1) { + perror(va_display.c_str()); + return NULL; + } + unique_ptr ret(new VADisplayWithCleanup); + ret->drm_fd = drm_fd; + ret->can_use_zerocopy = false; + ret->va_dpy = vaGetDisplayDRM(drm_fd); + if (ret->va_dpy == nullptr) { + return nullptr; + } + return ret; + } +} + +unique_ptr try_open_va(const string &va_display, VAProfile *h264_profile, string *error) +{ + unique_ptr va_dpy = va_open_display(va_display); + if (va_dpy == nullptr) { + if (error) *error = "Opening VA display failed"; + return nullptr; + } + int major_ver, minor_ver; + VAStatus va_status = vaInitialize(va_dpy->va_dpy, &major_ver, &minor_ver); + if (va_status != VA_STATUS_SUCCESS) { + char buf[256]; + snprintf(buf, sizeof(buf), "vaInitialize() failed with status %d\n", va_status); + if (error != nullptr) *error = buf; + return nullptr; + } + + int num_entrypoints = vaMaxNumEntrypoints(va_dpy->va_dpy); + unique_ptr entrypoints(new VAEntrypoint[num_entrypoints]); + if (entrypoints == nullptr) { + if (error != nullptr) *error = "Failed to allocate memory for VA entry points"; + return nullptr; + } + + // Try the profiles from highest to lowest until we find one that can be encoded. + constexpr VAProfile profile_list[] = { VAProfileH264High, VAProfileH264Main, VAProfileH264ConstrainedBaseline }; + for (unsigned i = 0; i < sizeof(profile_list) / sizeof(profile_list[0]); ++i) { + vaQueryConfigEntrypoints(va_dpy->va_dpy, profile_list[i], entrypoints.get(), &num_entrypoints); + for (int slice_entrypoint = 0; slice_entrypoint < num_entrypoints; slice_entrypoint++) { + if (entrypoints[slice_entrypoint] != VAEntrypointEncSlice) { + continue; + } + + // We found a usable encoder, so return it. + if (h264_profile != nullptr) { + *h264_profile = profile_list[i]; + } + return va_dpy; + } + } + + if (error != nullptr) *error = "Can't find VAEntrypointEncSlice for H264 profiles"; + return nullptr; +} + +int QuickSyncEncoderImpl::init_va(const string &va_display) +{ + string error; + va_dpy = try_open_va(va_display, &h264_profile, &error); + if (va_dpy == nullptr) { + fprintf(stderr, "error: %s\n", error.c_str()); + exit(1); + } + if (!va_dpy->can_use_zerocopy) { + use_zerocopy = false; + } + + switch (h264_profile) { + case VAProfileH264ConstrainedBaseline: + constraint_set_flag |= (1 << 0 | 1 << 1); /* Annex A.2.2 */ + ip_period = 1; + break; + + case VAProfileH264Main: + constraint_set_flag |= (1 << 1); /* Annex A.2.2 */ + break; + + case VAProfileH264High: + constraint_set_flag |= (1 << 3); /* Annex A.2.4 */ + break; + default: + h264_profile = VAProfileH264ConstrainedBaseline; + ip_period = 1; + constraint_set_flag |= (1 << 0); /* Annex A.2.1 */ + break; + } + + VAConfigAttrib attrib[VAConfigAttribTypeMax]; + + /* find out the format for the render target, and rate control mode */ + for (unsigned i = 0; i < VAConfigAttribTypeMax; i++) + attrib[i].type = (VAConfigAttribType)i; + + VAStatus va_status = vaGetConfigAttributes(va_dpy->va_dpy, h264_profile, VAEntrypointEncSlice, + &attrib[0], VAConfigAttribTypeMax); + CHECK_VASTATUS(va_status, "vaGetConfigAttributes"); + /* check the interested configattrib */ + if ((attrib[VAConfigAttribRTFormat].value & VA_RT_FORMAT_YUV420) == 0) { + printf("Not find desired YUV420 RT format\n"); + exit(1); + } else { + config_attrib[config_attrib_num].type = VAConfigAttribRTFormat; + config_attrib[config_attrib_num].value = VA_RT_FORMAT_YUV420; + config_attrib_num++; + } + + if (attrib[VAConfigAttribRateControl].value != VA_ATTRIB_NOT_SUPPORTED) { + if (!(attrib[VAConfigAttribRateControl].value & VA_RC_CQP)) { + fprintf(stderr, "ERROR: VA-API encoder does not support CQP mode.\n"); + exit(1); + } + + config_attrib[config_attrib_num].type = VAConfigAttribRateControl; + config_attrib[config_attrib_num].value = VA_RC_CQP; + config_attrib_num++; + } + + + if (attrib[VAConfigAttribEncPackedHeaders].value != VA_ATTRIB_NOT_SUPPORTED) { + int tmp = attrib[VAConfigAttribEncPackedHeaders].value; + + h264_packedheader = 1; + config_attrib[config_attrib_num].type = VAConfigAttribEncPackedHeaders; + config_attrib[config_attrib_num].value = VA_ENC_PACKED_HEADER_NONE; + + if (tmp & VA_ENC_PACKED_HEADER_SEQUENCE) { + config_attrib[config_attrib_num].value |= VA_ENC_PACKED_HEADER_SEQUENCE; + } + + if (tmp & VA_ENC_PACKED_HEADER_PICTURE) { + config_attrib[config_attrib_num].value |= VA_ENC_PACKED_HEADER_PICTURE; + } + + if (tmp & VA_ENC_PACKED_HEADER_SLICE) { + config_attrib[config_attrib_num].value |= VA_ENC_PACKED_HEADER_SLICE; + } + + if (tmp & VA_ENC_PACKED_HEADER_MISC) { + config_attrib[config_attrib_num].value |= VA_ENC_PACKED_HEADER_MISC; + } + + enc_packed_header_idx = config_attrib_num; + config_attrib_num++; + } + + if (attrib[VAConfigAttribEncInterlaced].value != VA_ATTRIB_NOT_SUPPORTED) { + config_attrib[config_attrib_num].type = VAConfigAttribEncInterlaced; + config_attrib[config_attrib_num].value = VA_ENC_PACKED_HEADER_NONE; + config_attrib_num++; + } + + if (attrib[VAConfigAttribEncMaxRefFrames].value != VA_ATTRIB_NOT_SUPPORTED) { + h264_maxref = attrib[VAConfigAttribEncMaxRefFrames].value; + } + + return 0; +} + +int QuickSyncEncoderImpl::setup_encode() +{ + if (!global_flags.x264_video_to_disk) { + VAStatus va_status; + VASurfaceID *tmp_surfaceid; + int codedbuf_size; + VASurfaceID src_surface[SURFACE_NUM]; + VASurfaceID ref_surface[SURFACE_NUM]; + + va_status = vaCreateConfig(va_dpy->va_dpy, h264_profile, VAEntrypointEncSlice, + &config_attrib[0], config_attrib_num, &config_id); + CHECK_VASTATUS(va_status, "vaCreateConfig"); + + /* create source surfaces */ + va_status = vaCreateSurfaces(va_dpy->va_dpy, + VA_RT_FORMAT_YUV420, frame_width_mbaligned, frame_height_mbaligned, + &src_surface[0], SURFACE_NUM, + NULL, 0); + CHECK_VASTATUS(va_status, "vaCreateSurfaces"); + + /* create reference surfaces */ + va_status = vaCreateSurfaces(va_dpy->va_dpy, + VA_RT_FORMAT_YUV420, frame_width_mbaligned, frame_height_mbaligned, + &ref_surface[0], SURFACE_NUM, + NULL, 0); + CHECK_VASTATUS(va_status, "vaCreateSurfaces"); + + tmp_surfaceid = (VASurfaceID *)calloc(2 * SURFACE_NUM, sizeof(VASurfaceID)); + memcpy(tmp_surfaceid, src_surface, SURFACE_NUM * sizeof(VASurfaceID)); + memcpy(tmp_surfaceid + SURFACE_NUM, ref_surface, SURFACE_NUM * sizeof(VASurfaceID)); + + for (int i = 0; i < SURFACE_NUM; i++) { + gl_surfaces[i].src_surface = src_surface[i]; + gl_surfaces[i].ref_surface = ref_surface[i]; + } + + /* Create a context for this encode pipe */ + va_status = vaCreateContext(va_dpy->va_dpy, config_id, + frame_width_mbaligned, frame_height_mbaligned, + VA_PROGRESSIVE, + tmp_surfaceid, 2 * SURFACE_NUM, + &context_id); + CHECK_VASTATUS(va_status, "vaCreateContext"); + free(tmp_surfaceid); + + codedbuf_size = (frame_width_mbaligned * frame_height_mbaligned * 400) / (16*16); + + for (int i = 0; i < SURFACE_NUM; i++) { + /* create coded buffer once for all + * other VA buffers which won't be used again after vaRenderPicture. + * so APP can always vaCreateBuffer for every frame + * but coded buffer need to be mapped and accessed after vaRenderPicture/vaEndPicture + * so VA won't maintain the coded buffer + */ + va_status = vaCreateBuffer(va_dpy->va_dpy, context_id, VAEncCodedBufferType, + codedbuf_size, 1, NULL, &gl_surfaces[i].coded_buf); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + } + } + + /* create OpenGL objects */ + for (int i = 0; i < SURFACE_NUM; i++) { + if (use_zerocopy) { + gl_surfaces[i].y_tex = resource_pool->create_2d_texture(GL_R8, 1, 1); + gl_surfaces[i].cbcr_tex = resource_pool->create_2d_texture(GL_RG8, 1, 1); + } else { + size_t bytes_per_pixel = (global_flags.x264_bit_depth > 8) ? 2 : 1; + + // Generate a PBO to read into. It doesn't necessarily fit 1:1 with the VA-API + // buffers, due to potentially differing pitch. + glGenBuffers(1, &gl_surfaces[i].pbo); + glBindBuffer(GL_PIXEL_PACK_BUFFER, gl_surfaces[i].pbo); + glBufferStorage(GL_PIXEL_PACK_BUFFER, frame_width * frame_height * 2 * bytes_per_pixel, nullptr, GL_MAP_READ_BIT | GL_MAP_WRITE_BIT | GL_MAP_PERSISTENT_BIT); + uint8_t *ptr = (uint8_t *)glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, frame_width * frame_height * 2 * bytes_per_pixel, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT); + gl_surfaces[i].y_offset = 0; + gl_surfaces[i].cbcr_offset = frame_width * frame_height * bytes_per_pixel; + gl_surfaces[i].y_ptr = ptr + gl_surfaces[i].y_offset; + gl_surfaces[i].cbcr_ptr = ptr + gl_surfaces[i].cbcr_offset; + glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); + } + } + + return 0; +} + +// Given a list like 1 9 3 0 2 8 4 and a pivot element 3, will produce +// +// 2 1 0 [3] 4 8 9 +template +static void sort_two(T *begin, T *end, const T &pivot, const C &less_than) +{ + T *middle = partition(begin, end, [&](const T &elem) { return less_than(elem, pivot); }); + sort(begin, middle, [&](const T &a, const T &b) { return less_than(b, a); }); + sort(middle, end, less_than); +} + +void QuickSyncEncoderImpl::update_ReferenceFrames(int current_display_frame, int frame_type) +{ + if (frame_type == FRAME_B) + return; + + pic_param.CurrPic.frame_idx = current_ref_frame_num; + + CurrentCurrPic.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; + unique_lock lock(storage_task_queue_mutex); + + // Insert the new frame at the start of the reference queue. + reference_frames.push_front(ReferenceFrame{ CurrentCurrPic, current_display_frame }); + + if (reference_frames.size() > num_ref_frames) + { + // The back frame frame is no longer in use as a reference. + int display_frame_num = reference_frames.back().display_number; + assert(surface_for_frame.count(display_frame_num)); + release_gl_surface(display_frame_num); + reference_frames.pop_back(); + } + + // Mark this frame in use as a reference. + assert(surface_for_frame.count(current_display_frame)); + ++surface_for_frame[current_display_frame]->refcount; + + current_ref_frame_num++; + if (current_ref_frame_num > MaxFrameNum) + current_ref_frame_num = 0; +} + + +void QuickSyncEncoderImpl::update_RefPicList_P(VAPictureH264 RefPicList0_P[MAX_NUM_REF2]) +{ + const auto descending_by_frame_idx = [](const VAPictureH264 &a, const VAPictureH264 &b) { + return a.frame_idx > b.frame_idx; + }; + + for (size_t i = 0; i < reference_frames.size(); ++i) { + RefPicList0_P[i] = reference_frames[i].pic; + } + sort(&RefPicList0_P[0], &RefPicList0_P[reference_frames.size()], descending_by_frame_idx); +} + +void QuickSyncEncoderImpl::update_RefPicList_B(VAPictureH264 RefPicList0_B[MAX_NUM_REF2], VAPictureH264 RefPicList1_B[MAX_NUM_REF2]) +{ + const auto ascending_by_top_field_order_cnt = [](const VAPictureH264 &a, const VAPictureH264 &b) { + return a.TopFieldOrderCnt < b.TopFieldOrderCnt; + }; + const auto descending_by_top_field_order_cnt = [](const VAPictureH264 &a, const VAPictureH264 &b) { + return a.TopFieldOrderCnt > b.TopFieldOrderCnt; + }; + + for (size_t i = 0; i < reference_frames.size(); ++i) { + RefPicList0_B[i] = reference_frames[i].pic; + RefPicList1_B[i] = reference_frames[i].pic; + } + sort_two(&RefPicList0_B[0], &RefPicList0_B[reference_frames.size()], CurrentCurrPic, ascending_by_top_field_order_cnt); + sort_two(&RefPicList1_B[0], &RefPicList1_B[reference_frames.size()], CurrentCurrPic, descending_by_top_field_order_cnt); +} + + +int QuickSyncEncoderImpl::render_sequence() +{ + VABufferID seq_param_buf, rc_param_buf, render_id[2]; + VAStatus va_status; + VAEncMiscParameterBuffer *misc_param; + VAEncMiscParameterRateControl *misc_rate_ctrl; + + seq_param.level_idc = 41 /*SH_LEVEL_3*/; + seq_param.picture_width_in_mbs = frame_width_mbaligned / 16; + seq_param.picture_height_in_mbs = frame_height_mbaligned / 16; + seq_param.bits_per_second = frame_bitrate; + + seq_param.intra_period = intra_period; + seq_param.intra_idr_period = intra_idr_period; + seq_param.ip_period = ip_period; + + seq_param.max_num_ref_frames = num_ref_frames; + seq_param.seq_fields.bits.frame_mbs_only_flag = 1; + seq_param.time_scale = TIMEBASE * 2; + seq_param.num_units_in_tick = 1; /* Tc = num_units_in_tick / scale */ + seq_param.seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4 = Log2MaxPicOrderCntLsb - 4; + seq_param.seq_fields.bits.log2_max_frame_num_minus4 = Log2MaxFrameNum - 4;; + seq_param.seq_fields.bits.frame_mbs_only_flag = 1; + seq_param.seq_fields.bits.chroma_format_idc = 1; + seq_param.seq_fields.bits.direct_8x8_inference_flag = 1; + + if (frame_width != frame_width_mbaligned || + frame_height != frame_height_mbaligned) { + seq_param.frame_cropping_flag = 1; + seq_param.frame_crop_left_offset = 0; + seq_param.frame_crop_right_offset = (frame_width_mbaligned - frame_width)/2; + seq_param.frame_crop_top_offset = 0; + seq_param.frame_crop_bottom_offset = (frame_height_mbaligned - frame_height)/2; + } + + va_status = vaCreateBuffer(va_dpy->va_dpy, context_id, + VAEncSequenceParameterBufferType, + sizeof(seq_param), 1, &seq_param, &seq_param_buf); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + va_status = vaCreateBuffer(va_dpy->va_dpy, context_id, + VAEncMiscParameterBufferType, + sizeof(VAEncMiscParameterBuffer) + sizeof(VAEncMiscParameterRateControl), + 1, NULL, &rc_param_buf); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + vaMapBuffer(va_dpy->va_dpy, rc_param_buf, (void **)&misc_param); + misc_param->type = VAEncMiscParameterTypeRateControl; + misc_rate_ctrl = (VAEncMiscParameterRateControl *)misc_param->data; + memset(misc_rate_ctrl, 0, sizeof(*misc_rate_ctrl)); + misc_rate_ctrl->bits_per_second = frame_bitrate; + misc_rate_ctrl->target_percentage = 66; + misc_rate_ctrl->window_size = 1000; + misc_rate_ctrl->initial_qp = initial_qp; + misc_rate_ctrl->min_qp = minimal_qp; + misc_rate_ctrl->basic_unit_size = 0; + vaUnmapBuffer(va_dpy->va_dpy, rc_param_buf); + + render_id[0] = seq_param_buf; + render_id[1] = rc_param_buf; + + render_picture_and_delete(va_dpy->va_dpy, context_id, &render_id[0], 2); + + return 0; +} + +static int calc_poc(int pic_order_cnt_lsb, int frame_type) +{ + static int PicOrderCntMsb_ref = 0, pic_order_cnt_lsb_ref = 0; + int prevPicOrderCntMsb, prevPicOrderCntLsb; + int PicOrderCntMsb, TopFieldOrderCnt; + + if (frame_type == FRAME_IDR) + prevPicOrderCntMsb = prevPicOrderCntLsb = 0; + else { + prevPicOrderCntMsb = PicOrderCntMsb_ref; + prevPicOrderCntLsb = pic_order_cnt_lsb_ref; + } + + if ((pic_order_cnt_lsb < prevPicOrderCntLsb) && + ((prevPicOrderCntLsb - pic_order_cnt_lsb) >= (int)(MaxPicOrderCntLsb / 2))) + PicOrderCntMsb = prevPicOrderCntMsb + MaxPicOrderCntLsb; + else if ((pic_order_cnt_lsb > prevPicOrderCntLsb) && + ((pic_order_cnt_lsb - prevPicOrderCntLsb) > (int)(MaxPicOrderCntLsb / 2))) + PicOrderCntMsb = prevPicOrderCntMsb - MaxPicOrderCntLsb; + else + PicOrderCntMsb = prevPicOrderCntMsb; + + TopFieldOrderCnt = PicOrderCntMsb + pic_order_cnt_lsb; + + if (frame_type != FRAME_B) { + PicOrderCntMsb_ref = PicOrderCntMsb; + pic_order_cnt_lsb_ref = pic_order_cnt_lsb; + } + + return TopFieldOrderCnt; +} + +int QuickSyncEncoderImpl::render_picture(GLSurface *surf, int frame_type, int display_frame_num, int gop_start_display_frame_num) +{ + VABufferID pic_param_buf; + VAStatus va_status; + size_t i = 0; + + pic_param.CurrPic.picture_id = surf->ref_surface; + pic_param.CurrPic.frame_idx = current_ref_frame_num; + pic_param.CurrPic.flags = 0; + pic_param.CurrPic.TopFieldOrderCnt = calc_poc((display_frame_num - gop_start_display_frame_num) % MaxPicOrderCntLsb, frame_type); + pic_param.CurrPic.BottomFieldOrderCnt = pic_param.CurrPic.TopFieldOrderCnt; + CurrentCurrPic = pic_param.CurrPic; + + for (i = 0; i < reference_frames.size(); i++) { + pic_param.ReferenceFrames[i] = reference_frames[i].pic; + } + for (i = reference_frames.size(); i < MAX_NUM_REF1; i++) { + pic_param.ReferenceFrames[i].picture_id = VA_INVALID_SURFACE; + pic_param.ReferenceFrames[i].flags = VA_PICTURE_H264_INVALID; + } + + pic_param.pic_fields.bits.idr_pic_flag = (frame_type == FRAME_IDR); + pic_param.pic_fields.bits.reference_pic_flag = (frame_type != FRAME_B); + pic_param.pic_fields.bits.entropy_coding_mode_flag = h264_entropy_mode; + pic_param.pic_fields.bits.deblocking_filter_control_present_flag = 1; + pic_param.frame_num = current_ref_frame_num; // FIXME: is this correct? + pic_param.coded_buf = surf->coded_buf; + pic_param.last_picture = false; // FIXME + pic_param.pic_init_qp = initial_qp; + + va_status = vaCreateBuffer(va_dpy->va_dpy, context_id, VAEncPictureParameterBufferType, + sizeof(pic_param), 1, &pic_param, &pic_param_buf); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + render_picture_and_delete(va_dpy->va_dpy, context_id, &pic_param_buf, 1); + + return 0; +} + +int QuickSyncEncoderImpl::render_packedsequence(YCbCrLumaCoefficients ycbcr_coefficients) +{ + VAEncPackedHeaderParameterBuffer packedheader_param_buffer; + VABufferID packedseq_para_bufid, packedseq_data_bufid, render_id[2]; + unsigned int length_in_bits; + unsigned char *packedseq_buffer = NULL; + VAStatus va_status; + + length_in_bits = build_packed_seq_buffer(ycbcr_coefficients, &packedseq_buffer); + + packedheader_param_buffer.type = VAEncPackedHeaderSequence; + + packedheader_param_buffer.bit_length = length_in_bits; /*length_in_bits*/ + packedheader_param_buffer.has_emulation_bytes = 0; + va_status = vaCreateBuffer(va_dpy->va_dpy, + context_id, + VAEncPackedHeaderParameterBufferType, + sizeof(packedheader_param_buffer), 1, &packedheader_param_buffer, + &packedseq_para_bufid); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + va_status = vaCreateBuffer(va_dpy->va_dpy, + context_id, + VAEncPackedHeaderDataBufferType, + (length_in_bits + 7) / 8, 1, packedseq_buffer, + &packedseq_data_bufid); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + render_id[0] = packedseq_para_bufid; + render_id[1] = packedseq_data_bufid; + render_picture_and_delete(va_dpy->va_dpy, context_id, render_id, 2); + + free(packedseq_buffer); + + return 0; +} + + +int QuickSyncEncoderImpl::render_packedpicture() +{ + VAEncPackedHeaderParameterBuffer packedheader_param_buffer; + VABufferID packedpic_para_bufid, packedpic_data_bufid, render_id[2]; + unsigned int length_in_bits; + unsigned char *packedpic_buffer = NULL; + VAStatus va_status; + + length_in_bits = build_packed_pic_buffer(&packedpic_buffer); + packedheader_param_buffer.type = VAEncPackedHeaderPicture; + packedheader_param_buffer.bit_length = length_in_bits; + packedheader_param_buffer.has_emulation_bytes = 0; + + va_status = vaCreateBuffer(va_dpy->va_dpy, + context_id, + VAEncPackedHeaderParameterBufferType, + sizeof(packedheader_param_buffer), 1, &packedheader_param_buffer, + &packedpic_para_bufid); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + va_status = vaCreateBuffer(va_dpy->va_dpy, + context_id, + VAEncPackedHeaderDataBufferType, + (length_in_bits + 7) / 8, 1, packedpic_buffer, + &packedpic_data_bufid); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + render_id[0] = packedpic_para_bufid; + render_id[1] = packedpic_data_bufid; + render_picture_and_delete(va_dpy->va_dpy, context_id, render_id, 2); + + free(packedpic_buffer); + + return 0; +} + +void QuickSyncEncoderImpl::render_packedslice() +{ + VAEncPackedHeaderParameterBuffer packedheader_param_buffer; + VABufferID packedslice_para_bufid, packedslice_data_bufid, render_id[2]; + unsigned int length_in_bits; + unsigned char *packedslice_buffer = NULL; + VAStatus va_status; + + length_in_bits = build_packed_slice_buffer(&packedslice_buffer); + packedheader_param_buffer.type = VAEncPackedHeaderSlice; + packedheader_param_buffer.bit_length = length_in_bits; + packedheader_param_buffer.has_emulation_bytes = 0; + + va_status = vaCreateBuffer(va_dpy->va_dpy, + context_id, + VAEncPackedHeaderParameterBufferType, + sizeof(packedheader_param_buffer), 1, &packedheader_param_buffer, + &packedslice_para_bufid); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + va_status = vaCreateBuffer(va_dpy->va_dpy, + context_id, + VAEncPackedHeaderDataBufferType, + (length_in_bits + 7) / 8, 1, packedslice_buffer, + &packedslice_data_bufid); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + render_id[0] = packedslice_para_bufid; + render_id[1] = packedslice_data_bufid; + render_picture_and_delete(va_dpy->va_dpy, context_id, render_id, 2); + + free(packedslice_buffer); +} + +int QuickSyncEncoderImpl::render_slice(int encoding_frame_num, int display_frame_num, int gop_start_display_frame_num, int frame_type) +{ + VABufferID slice_param_buf; + VAStatus va_status; + int i; + + /* one frame, one slice */ + slice_param.macroblock_address = 0; + slice_param.num_macroblocks = frame_width_mbaligned * frame_height_mbaligned/(16*16); /* Measured by MB */ + slice_param.slice_type = (frame_type == FRAME_IDR)?2:frame_type; + if (frame_type == FRAME_IDR) { + if (encoding_frame_num != 0) + ++slice_param.idr_pic_id; + } else if (frame_type == FRAME_P) { + VAPictureH264 RefPicList0_P[MAX_NUM_REF2]; + update_RefPicList_P(RefPicList0_P); + + int refpiclist0_max = h264_maxref & 0xffff; + memcpy(slice_param.RefPicList0, RefPicList0_P, refpiclist0_max*sizeof(VAPictureH264)); + + for (i = refpiclist0_max; i < MAX_NUM_REF2; i++) { + slice_param.RefPicList0[i].picture_id = VA_INVALID_SURFACE; + slice_param.RefPicList0[i].flags = VA_PICTURE_H264_INVALID; + } + } else if (frame_type == FRAME_B) { + VAPictureH264 RefPicList0_B[MAX_NUM_REF2], RefPicList1_B[MAX_NUM_REF2]; + update_RefPicList_B(RefPicList0_B, RefPicList1_B); + + int refpiclist0_max = h264_maxref & 0xffff; + int refpiclist1_max = (h264_maxref >> 16) & 0xffff; + + memcpy(slice_param.RefPicList0, RefPicList0_B, refpiclist0_max*sizeof(VAPictureH264)); + for (i = refpiclist0_max; i < MAX_NUM_REF2; i++) { + slice_param.RefPicList0[i].picture_id = VA_INVALID_SURFACE; + slice_param.RefPicList0[i].flags = VA_PICTURE_H264_INVALID; + } + + memcpy(slice_param.RefPicList1, RefPicList1_B, refpiclist1_max*sizeof(VAPictureH264)); + for (i = refpiclist1_max; i < MAX_NUM_REF2; i++) { + slice_param.RefPicList1[i].picture_id = VA_INVALID_SURFACE; + slice_param.RefPicList1[i].flags = VA_PICTURE_H264_INVALID; + } + } + + slice_param.slice_alpha_c0_offset_div2 = 0; + slice_param.slice_beta_offset_div2 = 0; + slice_param.direct_spatial_mv_pred_flag = 1; + slice_param.pic_order_cnt_lsb = (display_frame_num - gop_start_display_frame_num) % MaxPicOrderCntLsb; + + + if (h264_packedheader && + config_attrib[enc_packed_header_idx].value & VA_ENC_PACKED_HEADER_SLICE) + render_packedslice(); + + va_status = vaCreateBuffer(va_dpy->va_dpy, context_id, VAEncSliceParameterBufferType, + sizeof(slice_param), 1, &slice_param, &slice_param_buf); + CHECK_VASTATUS(va_status, "vaCreateBuffer"); + + render_picture_and_delete(va_dpy->va_dpy, context_id, &slice_param_buf, 1); + + return 0; +} + + + +void QuickSyncEncoderImpl::save_codeddata(GLSurface *surf, storage_task task) +{ + VACodedBufferSegment *buf_list = NULL; + VAStatus va_status; + + string data; + + va_status = vaMapBuffer(va_dpy->va_dpy, surf->coded_buf, (void **)(&buf_list)); + CHECK_VASTATUS(va_status, "vaMapBuffer"); + while (buf_list != NULL) { + data.append(reinterpret_cast(buf_list->buf), buf_list->size); + buf_list = (VACodedBufferSegment *) buf_list->next; + } + vaUnmapBuffer(va_dpy->va_dpy, surf->coded_buf); + + static int frameno = 0; + print_latency("Current Quick Sync latency (video inputs → disk mux):", + task.received_ts, (task.frame_type == FRAME_B), &frameno, &qs_latency_histogram); + + { + // Add video. + AVPacket pkt; + memset(&pkt, 0, sizeof(pkt)); + pkt.buf = nullptr; + pkt.data = reinterpret_cast(&data[0]); + pkt.size = data.size(); + pkt.stream_index = 0; + if (task.frame_type == FRAME_IDR) { + pkt.flags = AV_PKT_FLAG_KEY; + } else { + pkt.flags = 0; + } + pkt.duration = task.duration; + if (file_mux) { + file_mux->add_packet(pkt, task.pts + global_delay(), task.dts + global_delay()); + } + if (!global_flags.uncompressed_video_to_http && + !global_flags.x264_video_to_http) { + stream_mux->add_packet(pkt, task.pts + global_delay(), task.dts + global_delay()); + } + } +} + + +// this is weird. but it seems to put a new frame onto the queue +void QuickSyncEncoderImpl::storage_task_enqueue(storage_task task) +{ + unique_lock lock(storage_task_queue_mutex); + storage_task_queue.push(move(task)); + storage_task_queue_changed.notify_all(); +} + +void QuickSyncEncoderImpl::storage_task_thread() +{ + pthread_setname_np(pthread_self(), "QS_Storage"); + for ( ;; ) { + storage_task current; + GLSurface *surf; + { + // wait until there's an encoded frame + unique_lock lock(storage_task_queue_mutex); + storage_task_queue_changed.wait(lock, [this]{ return storage_thread_should_quit || !storage_task_queue.empty(); }); + if (storage_thread_should_quit && storage_task_queue.empty()) return; + current = move(storage_task_queue.front()); + storage_task_queue.pop(); + surf = surface_for_frame[current.display_order]; + assert(surf != nullptr); + } + + VAStatus va_status; + + size_t display_order = current.display_order; + vector ref_display_frame_numbers = move(current.ref_display_frame_numbers); + + // waits for data, then saves it to disk. + va_status = vaSyncSurface(va_dpy->va_dpy, surf->src_surface); + CHECK_VASTATUS(va_status, "vaSyncSurface"); + save_codeddata(surf, move(current)); + + // Unlock the frame, and all its references. + { + unique_lock lock(storage_task_queue_mutex); + release_gl_surface(display_order); + + for (size_t frame_num : ref_display_frame_numbers) { + release_gl_surface(frame_num); + } + } + } +} + +void QuickSyncEncoderImpl::release_encode() +{ + for (unsigned i = 0; i < SURFACE_NUM; i++) { + vaDestroyBuffer(va_dpy->va_dpy, gl_surfaces[i].coded_buf); + vaDestroySurfaces(va_dpy->va_dpy, &gl_surfaces[i].src_surface, 1); + vaDestroySurfaces(va_dpy->va_dpy, &gl_surfaces[i].ref_surface, 1); + } + + vaDestroyContext(va_dpy->va_dpy, context_id); + vaDestroyConfig(va_dpy->va_dpy, config_id); +} + +void QuickSyncEncoderImpl::release_gl_resources() +{ + assert(is_shutdown); + if (has_released_gl_resources) { + return; + } + + for (unsigned i = 0; i < SURFACE_NUM; i++) { + if (use_zerocopy) { + resource_pool->release_2d_texture(gl_surfaces[i].y_tex); + resource_pool->release_2d_texture(gl_surfaces[i].cbcr_tex); + } else { + glBindBuffer(GL_PIXEL_PACK_BUFFER, gl_surfaces[i].pbo); + glUnmapBuffer(GL_PIXEL_PACK_BUFFER); + glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); + glDeleteBuffers(1, &gl_surfaces[i].pbo); + } + } + + has_released_gl_resources = true; +} + +QuickSyncEncoderImpl::QuickSyncEncoderImpl(const std::string &filename, ResourcePool *resource_pool, QSurface *surface, const string &va_display, int width, int height, AVOutputFormat *oformat, X264Encoder *x264_encoder, DiskSpaceEstimator *disk_space_estimator) + : current_storage_frame(0), resource_pool(resource_pool), surface(surface), x264_encoder(x264_encoder), frame_width(width), frame_height(height), disk_space_estimator(disk_space_estimator) +{ + file_audio_encoder.reset(new AudioEncoder(AUDIO_OUTPUT_CODEC_NAME, DEFAULT_AUDIO_OUTPUT_BIT_RATE, oformat)); + open_output_file(filename); + file_audio_encoder->add_mux(file_mux.get()); + + frame_width_mbaligned = (frame_width + 15) & (~15); + frame_height_mbaligned = (frame_height + 15) & (~15); + + //print_input(); + + if (global_flags.x264_video_to_http || global_flags.x264_video_to_disk) { + assert(x264_encoder != nullptr); + } else { + assert(x264_encoder == nullptr); + } + + enable_zerocopy_if_possible(); + if (!global_flags.x264_video_to_disk) { + init_va(va_display); + } + setup_encode(); + + if (!global_flags.x264_video_to_disk) { + memset(&seq_param, 0, sizeof(seq_param)); + memset(&pic_param, 0, sizeof(pic_param)); + memset(&slice_param, 0, sizeof(slice_param)); + } + + call_once(quick_sync_metrics_inited, [](){ + mixer_latency_histogram.init("mixer"); + qs_latency_histogram.init("quick_sync"); + current_file_mux_metrics.init({{ "destination", "current_file" }}); + total_mux_metrics.init({{ "destination", "files_total" }}); + global_metrics.add("current_file_start_time_seconds", &metric_current_file_start_time_seconds, Metrics::TYPE_GAUGE); + global_metrics.add("quick_sync_stalled_frames", &metric_quick_sync_stalled_frames); + }); + + storage_thread = thread(&QuickSyncEncoderImpl::storage_task_thread, this); + + encode_thread = thread([this]{ + QOpenGLContext *context = create_context(this->surface); + eglBindAPI(EGL_OPENGL_API); + if (!make_current(context, this->surface)) { + printf("display=%p surface=%p context=%p curr=%p err=%d\n", eglGetCurrentDisplay(), this->surface, context, eglGetCurrentContext(), + eglGetError()); + exit(1); + } + encode_thread_func(); + delete_context(context); + }); +} + +QuickSyncEncoderImpl::~QuickSyncEncoderImpl() +{ + shutdown(); + release_gl_resources(); +} + +QuickSyncEncoderImpl::GLSurface *QuickSyncEncoderImpl::allocate_gl_surface() +{ + for (unsigned i = 0; i < SURFACE_NUM; ++i) { + if (gl_surfaces[i].refcount == 0) { + ++gl_surfaces[i].refcount; + return &gl_surfaces[i]; + } + } + return nullptr; +} + +void QuickSyncEncoderImpl::release_gl_surface(size_t display_frame_num) +{ + assert(surface_for_frame.count(display_frame_num)); + QuickSyncEncoderImpl::GLSurface *surf = surface_for_frame[display_frame_num]; + if (--surf->refcount == 0) { + assert(surface_for_frame.count(display_frame_num)); + surface_for_frame.erase(display_frame_num); + storage_task_queue_changed.notify_all(); + } +} + +bool QuickSyncEncoderImpl::is_zerocopy() const +{ + return use_zerocopy; +} + +bool QuickSyncEncoderImpl::begin_frame(int64_t pts, int64_t duration, YCbCrLumaCoefficients ycbcr_coefficients, const vector &input_frames, GLuint *y_tex, GLuint *cbcr_tex) +{ + assert(!is_shutdown); + GLSurface *surf = nullptr; + { + // Wait until this frame slot is done encoding. + unique_lock lock(storage_task_queue_mutex); + surf = allocate_gl_surface(); + if (surf == nullptr) { + fprintf(stderr, "Warning: No free slots for frame %d, rendering has to wait for H.264 encoder\n", + current_storage_frame); + ++metric_quick_sync_stalled_frames; + storage_task_queue_changed.wait(lock, [this, &surf]{ + if (storage_thread_should_quit) + return true; + surf = allocate_gl_surface(); + return surf != nullptr; + }); + } + if (storage_thread_should_quit) return false; + assert(surf != nullptr); + surface_for_frame[current_storage_frame] = surf; + } + + if (use_zerocopy) { + *y_tex = surf->y_tex; + *cbcr_tex = surf->cbcr_tex; + } else { + surf->y_tex = *y_tex; + surf->cbcr_tex = *cbcr_tex; + } + + if (!global_flags.x264_video_to_disk) { + VAStatus va_status = vaDeriveImage(va_dpy->va_dpy, surf->src_surface, &surf->surface_image); + CHECK_VASTATUS(va_status, "vaDeriveImage"); + + if (use_zerocopy) { + VABufferInfo buf_info; + buf_info.mem_type = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME; // or VA_SURFACE_ATTRIB_MEM_TYPE_KERNEL_DRM? + va_status = vaAcquireBufferHandle(va_dpy->va_dpy, surf->surface_image.buf, &buf_info); + CHECK_VASTATUS(va_status, "vaAcquireBufferHandle"); + + // Create Y image. + surf->y_egl_image = EGL_NO_IMAGE_KHR; + EGLint y_attribs[] = { + EGL_WIDTH, frame_width, + EGL_HEIGHT, frame_height, + EGL_LINUX_DRM_FOURCC_EXT, fourcc_code('R', '8', ' ', ' '), + EGL_DMA_BUF_PLANE0_FD_EXT, EGLint(buf_info.handle), + EGL_DMA_BUF_PLANE0_OFFSET_EXT, EGLint(surf->surface_image.offsets[0]), + EGL_DMA_BUF_PLANE0_PITCH_EXT, EGLint(surf->surface_image.pitches[0]), + EGL_NONE + }; + + surf->y_egl_image = eglCreateImageKHR(eglGetCurrentDisplay(), EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, NULL, y_attribs); + assert(surf->y_egl_image != EGL_NO_IMAGE_KHR); + + // Associate Y image to a texture. + glBindTexture(GL_TEXTURE_2D, *y_tex); + glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, surf->y_egl_image); + + // Create CbCr image. + surf->cbcr_egl_image = EGL_NO_IMAGE_KHR; + EGLint cbcr_attribs[] = { + EGL_WIDTH, frame_width / 2, + EGL_HEIGHT, frame_height / 2, + EGL_LINUX_DRM_FOURCC_EXT, fourcc_code('G', 'R', '8', '8'), + EGL_DMA_BUF_PLANE0_FD_EXT, EGLint(buf_info.handle), + EGL_DMA_BUF_PLANE0_OFFSET_EXT, EGLint(surf->surface_image.offsets[1]), + EGL_DMA_BUF_PLANE0_PITCH_EXT, EGLint(surf->surface_image.pitches[1]), + EGL_NONE + }; + + surf->cbcr_egl_image = eglCreateImageKHR(eglGetCurrentDisplay(), EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, NULL, cbcr_attribs); + assert(surf->cbcr_egl_image != EGL_NO_IMAGE_KHR); + + // Associate CbCr image to a texture. + glBindTexture(GL_TEXTURE_2D, *cbcr_tex); + glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, surf->cbcr_egl_image); + } + } + + current_video_frame = PendingFrame{ {}, input_frames, pts, duration, ycbcr_coefficients }; + + return true; +} + +void QuickSyncEncoderImpl::add_audio(int64_t pts, vector audio) +{ + lock_guard lock(file_audio_encoder_mutex); + assert(!is_shutdown); + file_audio_encoder->encode_audio(audio, pts + global_delay()); +} + +RefCountedGLsync QuickSyncEncoderImpl::end_frame() +{ + assert(!is_shutdown); + + if (!use_zerocopy) { + GLenum type = global_flags.x264_bit_depth > 8 ? GL_UNSIGNED_SHORT : GL_UNSIGNED_BYTE; + GLSurface *surf; + { + unique_lock lock(storage_task_queue_mutex); + surf = surface_for_frame[current_storage_frame]; + assert(surf != nullptr); + } + + glPixelStorei(GL_PACK_ROW_LENGTH, 0); + check_error(); + + glBindBuffer(GL_PIXEL_PACK_BUFFER, surf->pbo); + check_error(); + + glBindTexture(GL_TEXTURE_2D, surf->y_tex); + check_error(); + glGetTexImage(GL_TEXTURE_2D, 0, GL_RED, type, BUFFER_OFFSET(surf->y_offset)); + check_error(); + + glBindTexture(GL_TEXTURE_2D, surf->cbcr_tex); + check_error(); + glGetTexImage(GL_TEXTURE_2D, 0, GL_RG, type, BUFFER_OFFSET(surf->cbcr_offset)); + check_error(); + + // We don't own these; the caller does. + surf->y_tex = surf->cbcr_tex = 0; + + glBindTexture(GL_TEXTURE_2D, 0); + check_error(); + glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); + check_error(); + + glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT | GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT); + check_error(); + } + + RefCountedGLsync fence = RefCountedGLsync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0); + check_error(); + glFlush(); // Make the H.264 thread see the fence as soon as possible. + check_error(); + + { + unique_lock lock(frame_queue_mutex); + current_video_frame.fence = fence; + pending_video_frames.push(move(current_video_frame)); + ++current_storage_frame; + } + frame_queue_nonempty.notify_all(); + return fence; +} + +void QuickSyncEncoderImpl::shutdown() +{ + if (is_shutdown) { + return; + } + + { + unique_lock lock(frame_queue_mutex); + encode_thread_should_quit = true; + frame_queue_nonempty.notify_all(); + } + encode_thread.join(); + { + unique_lock lock(storage_task_queue_mutex); + storage_thread_should_quit = true; + frame_queue_nonempty.notify_all(); + storage_task_queue_changed.notify_all(); + } + storage_thread.join(); + + // Encode any leftover audio in the queues, and also any delayed frames. + { + lock_guard lock(file_audio_encoder_mutex); + file_audio_encoder->encode_last_audio(); + } + + if (!global_flags.x264_video_to_disk) { + release_encode(); + va_dpy.reset(); + } + is_shutdown = true; +} + +void QuickSyncEncoderImpl::close_file() +{ + file_mux.reset(); + metric_current_file_start_time_seconds = 0.0 / 0.0; +} + +void QuickSyncEncoderImpl::open_output_file(const std::string &filename) +{ + AVFormatContext *avctx = avformat_alloc_context(); + avctx->oformat = av_guess_format(NULL, filename.c_str(), NULL); + assert(filename.size() < sizeof(avctx->filename) - 1); + strcpy(avctx->filename, filename.c_str()); + + string url = "file:" + filename; + int ret = avio_open2(&avctx->pb, url.c_str(), AVIO_FLAG_WRITE, &avctx->interrupt_callback, NULL); + if (ret < 0) { + char tmp[AV_ERROR_MAX_STRING_SIZE]; + fprintf(stderr, "%s: avio_open2() failed: %s\n", filename.c_str(), av_make_error_string(tmp, sizeof(tmp), ret)); + exit(1); + } + + string video_extradata; // FIXME: See other comment about global headers. + if (global_flags.x264_video_to_disk) { + video_extradata = x264_encoder->get_global_headers(); + } + + current_file_mux_metrics.reset(); + + { + lock_guard lock(file_audio_encoder_mutex); + AVCodecParametersWithDeleter audio_codecpar = file_audio_encoder->get_codec_parameters(); + file_mux.reset(new Mux(avctx, frame_width, frame_height, Mux::CODEC_H264, video_extradata, audio_codecpar.get(), TIMEBASE, + std::bind(&DiskSpaceEstimator::report_write, disk_space_estimator, filename, _1), + Mux::WRITE_BACKGROUND, + { ¤t_file_mux_metrics, &total_mux_metrics })); + } + metric_current_file_start_time_seconds = get_timestamp_for_metrics(); + + if (global_flags.x264_video_to_disk) { + x264_encoder->add_mux(file_mux.get()); + } +} + +void QuickSyncEncoderImpl::encode_thread_func() +{ + pthread_setname_np(pthread_self(), "QS_Encode"); + + int64_t last_dts = -1; + int gop_start_display_frame_num = 0; + for (int display_frame_num = 0; ; ++display_frame_num) { + // Wait for the frame to be in the queue. Note that this only means + // we started rendering it. + PendingFrame frame; + { + unique_lock lock(frame_queue_mutex); + frame_queue_nonempty.wait(lock, [this]{ + return encode_thread_should_quit || !pending_video_frames.empty(); + }); + if (encode_thread_should_quit && pending_video_frames.empty()) { + // We may have queued frames left in the reorder buffer + // that were supposed to be B-frames, but have no P-frame + // to be encoded against. If so, encode them all as + // P-frames instead. Note that this happens under the mutex, + // but nobody else uses it at this point, since we're shutting down, + // so there's no contention. + encode_remaining_frames_as_p(quicksync_encoding_frame_num, gop_start_display_frame_num, last_dts); + return; + } else { + frame = move(pending_video_frames.front()); + pending_video_frames.pop(); + } + } + + // Pass the frame on to x264 (or uncompressed to HTTP) as needed. + // Note that this implicitly waits for the frame to be done rendering. + pass_frame(frame, display_frame_num, frame.pts, frame.duration); + + if (global_flags.x264_video_to_disk) { + unique_lock lock(storage_task_queue_mutex); + release_gl_surface(display_frame_num); + continue; + } + + reorder_buffer[display_frame_num] = move(frame); + + // Now encode as many QuickSync frames as we can using the frames we have available. + // (It could be zero, or it could be multiple.) FIXME: make a function. + for ( ;; ) { + int pts_lag; + int frame_type, quicksync_display_frame_num; + encoding2display_order(quicksync_encoding_frame_num, intra_period, intra_idr_period, ip_period, + &quicksync_display_frame_num, &frame_type, &pts_lag); + if (!reorder_buffer.count(quicksync_display_frame_num)) { + break; + } + frame = move(reorder_buffer[quicksync_display_frame_num]); + reorder_buffer.erase(quicksync_display_frame_num); + + if (frame_type == FRAME_IDR) { + // Release any reference frames from the previous GOP. + { + unique_lock lock(storage_task_queue_mutex); + for (const ReferenceFrame &frame : reference_frames) { + release_gl_surface(frame.display_number); + } + } + reference_frames.clear(); + current_ref_frame_num = 0; + gop_start_display_frame_num = quicksync_display_frame_num; + } + + // Determine the dts of this frame. + int64_t dts; + if (pts_lag == -1) { + assert(last_dts != -1); + dts = last_dts + (TIMEBASE / MAX_FPS); + } else { + dts = frame.pts - pts_lag; + } + last_dts = dts; + + encode_frame(frame, quicksync_encoding_frame_num, quicksync_display_frame_num, gop_start_display_frame_num, frame_type, frame.pts, dts, frame.duration, frame.ycbcr_coefficients); + ++quicksync_encoding_frame_num; + } + } +} + +void QuickSyncEncoderImpl::encode_remaining_frames_as_p(int encoding_frame_num, int gop_start_display_frame_num, int64_t last_dts) +{ + if (reorder_buffer.empty()) { + return; + } + + for (auto &pending_frame : reorder_buffer) { + int display_frame_num = pending_frame.first; + assert(display_frame_num > 0); + PendingFrame frame = move(pending_frame.second); + int64_t dts = last_dts + (TIMEBASE / MAX_FPS); + printf("Finalizing encode: Encoding leftover frame %d as P-frame instead of B-frame.\n", display_frame_num); + encode_frame(frame, encoding_frame_num++, display_frame_num, gop_start_display_frame_num, FRAME_P, frame.pts, dts, frame.duration, frame.ycbcr_coefficients); + last_dts = dts; + } +} + +void QuickSyncEncoderImpl::add_packet_for_uncompressed_frame(int64_t pts, int64_t duration, const uint8_t *data) +{ + AVPacket pkt; + memset(&pkt, 0, sizeof(pkt)); + pkt.buf = nullptr; + pkt.data = const_cast(data); + pkt.size = frame_width * frame_height * 2; + pkt.stream_index = 0; + pkt.flags = AV_PKT_FLAG_KEY; + pkt.duration = duration; + stream_mux->add_packet(pkt, pts, pts); +} + +namespace { + +void memcpy_with_pitch(uint8_t *dst, const uint8_t *src, size_t src_width, size_t dst_pitch, size_t height) +{ + if (src_width == dst_pitch) { + memcpy(dst, src, src_width * height); + } else { + for (size_t y = 0; y < height; ++y) { + const uint8_t *sptr = src + y * src_width; + uint8_t *dptr = dst + y * dst_pitch; + memcpy(dptr, sptr, src_width); + } + } +} + +} // namespace + +void QuickSyncEncoderImpl::pass_frame(QuickSyncEncoderImpl::PendingFrame frame, int display_frame_num, int64_t pts, int64_t duration) +{ + // Wait for the GPU to be done with the frame. + GLenum sync_status; + do { + sync_status = glClientWaitSync(frame.fence.get(), 0, 0); + check_error(); + if (sync_status == GL_TIMEOUT_EXPIRED) { + // NVIDIA likes to busy-wait; yield instead. + this_thread::sleep_for(milliseconds(1)); + } + } while (sync_status == GL_TIMEOUT_EXPIRED); + assert(sync_status != GL_WAIT_FAILED); + + ReceivedTimestamps received_ts = find_received_timestamp(frame.input_frames); + static int frameno = 0; + print_latency("Current mixer latency (video inputs → ready for encode):", + received_ts, false, &frameno, &mixer_latency_histogram); + + // Release back any input frames we needed to render this frame. + frame.input_frames.clear(); + + GLSurface *surf; + { + unique_lock lock(storage_task_queue_mutex); + surf = surface_for_frame[display_frame_num]; + assert(surf != nullptr); + } + uint8_t *data = reinterpret_cast(surf->y_ptr); + if (global_flags.uncompressed_video_to_http) { + add_packet_for_uncompressed_frame(pts, duration, data); + } else if (global_flags.x264_video_to_http || global_flags.x264_video_to_disk) { + x264_encoder->add_frame(pts, duration, frame.ycbcr_coefficients, data, received_ts); + } +} + +void QuickSyncEncoderImpl::encode_frame(QuickSyncEncoderImpl::PendingFrame frame, int encoding_frame_num, int display_frame_num, int gop_start_display_frame_num, + int frame_type, int64_t pts, int64_t dts, int64_t duration, YCbCrLumaCoefficients ycbcr_coefficients) +{ + const ReceivedTimestamps received_ts = find_received_timestamp(frame.input_frames); + + GLSurface *surf; + { + unique_lock lock(storage_task_queue_mutex); + surf = surface_for_frame[display_frame_num]; + assert(surf != nullptr); + } + VAStatus va_status; + + if (use_zerocopy) { + eglDestroyImageKHR(eglGetCurrentDisplay(), surf->y_egl_image); + eglDestroyImageKHR(eglGetCurrentDisplay(), surf->cbcr_egl_image); + va_status = vaReleaseBufferHandle(va_dpy->va_dpy, surf->surface_image.buf); + CHECK_VASTATUS(va_status, "vaReleaseBufferHandle"); + } else { + // Upload the frame to VA-API. + unsigned char *surface_p = nullptr; + vaMapBuffer(va_dpy->va_dpy, surf->surface_image.buf, (void **)&surface_p); + + unsigned char *va_y_ptr = (unsigned char *)surface_p + surf->surface_image.offsets[0]; + memcpy_with_pitch(va_y_ptr, surf->y_ptr, frame_width, surf->surface_image.pitches[0], frame_height); + + unsigned char *va_cbcr_ptr = (unsigned char *)surface_p + surf->surface_image.offsets[1]; + memcpy_with_pitch(va_cbcr_ptr, surf->cbcr_ptr, (frame_width / 2) * sizeof(uint16_t), surf->surface_image.pitches[1], frame_height / 2); + + va_status = vaUnmapBuffer(va_dpy->va_dpy, surf->surface_image.buf); + CHECK_VASTATUS(va_status, "vaUnmapBuffer"); + } + + va_status = vaDestroyImage(va_dpy->va_dpy, surf->surface_image.image_id); + CHECK_VASTATUS(va_status, "vaDestroyImage"); + + // Schedule the frame for encoding. + VASurfaceID va_surface = surf->src_surface; + va_status = vaBeginPicture(va_dpy->va_dpy, context_id, va_surface); + CHECK_VASTATUS(va_status, "vaBeginPicture"); + + if (frame_type == FRAME_IDR) { + // FIXME: If the mux wants global headers, we should not put the + // SPS/PPS before each IDR frame, but rather put it into the + // codec extradata (formatted differently?). + // + // NOTE: If we change ycbcr_coefficients, it will not take effect + // before the next IDR frame. This is acceptable, as it should only + // happen on a mode change, which is rare. + render_sequence(); + render_picture(surf, frame_type, display_frame_num, gop_start_display_frame_num); + if (h264_packedheader) { + render_packedsequence(ycbcr_coefficients); + render_packedpicture(); + } + } else { + //render_sequence(); + render_picture(surf, frame_type, display_frame_num, gop_start_display_frame_num); + } + render_slice(encoding_frame_num, display_frame_num, gop_start_display_frame_num, frame_type); + + va_status = vaEndPicture(va_dpy->va_dpy, context_id); + CHECK_VASTATUS(va_status, "vaEndPicture"); + + update_ReferenceFrames(display_frame_num, frame_type); + + vector ref_display_frame_numbers; + + // Lock the references for this frame; otherwise, they could be + // rendered to before this frame is done encoding. + { + unique_lock lock(storage_task_queue_mutex); + for (const ReferenceFrame &frame : reference_frames) { + assert(surface_for_frame.count(frame.display_number)); + ++surface_for_frame[frame.display_number]->refcount; + ref_display_frame_numbers.push_back(frame.display_number); + } + } + + // so now the data is done encoding (well, async job kicked off)... + // we send that to the storage thread + storage_task tmp; + tmp.display_order = display_frame_num; + tmp.frame_type = frame_type; + tmp.pts = pts; + tmp.dts = dts; + tmp.duration = duration; + tmp.ycbcr_coefficients = ycbcr_coefficients; + tmp.received_ts = received_ts; + tmp.ref_display_frame_numbers = move(ref_display_frame_numbers); + storage_task_enqueue(move(tmp)); +} + +// Proxy object. +QuickSyncEncoder::QuickSyncEncoder(const std::string &filename, ResourcePool *resource_pool, QSurface *surface, const string &va_display, int width, int height, AVOutputFormat *oformat, X264Encoder *x264_encoder, DiskSpaceEstimator *disk_space_estimator) + : impl(new QuickSyncEncoderImpl(filename, resource_pool, surface, va_display, width, height, oformat, x264_encoder, disk_space_estimator)) {} + +// Must be defined here because unique_ptr<> destructor needs to know the impl. +QuickSyncEncoder::~QuickSyncEncoder() {} + +void QuickSyncEncoder::add_audio(int64_t pts, vector audio) +{ + impl->add_audio(pts, audio); +} + +bool QuickSyncEncoder::is_zerocopy() const +{ + return impl->is_zerocopy(); +} + +bool QuickSyncEncoder::begin_frame(int64_t pts, int64_t duration, YCbCrLumaCoefficients ycbcr_coefficients, const vector &input_frames, GLuint *y_tex, GLuint *cbcr_tex) +{ + return impl->begin_frame(pts, duration, ycbcr_coefficients, input_frames, y_tex, cbcr_tex); +} + +RefCountedGLsync QuickSyncEncoder::end_frame() +{ + return impl->end_frame(); +} + +void QuickSyncEncoder::shutdown() +{ + impl->shutdown(); +} + +void QuickSyncEncoder::close_file() +{ + impl->shutdown(); +} + +void QuickSyncEncoder::set_stream_mux(Mux *mux) +{ + impl->set_stream_mux(mux); +} + +int64_t QuickSyncEncoder::global_delay() const { + return impl->global_delay(); +} + +string QuickSyncEncoder::get_usable_va_display() +{ + // Reduce the amount of chatter while probing, + // unless the user has specified otherwise. + bool need_env_reset = false; + if (getenv("LIBVA_MESSAGING_LEVEL") == nullptr) { + setenv("LIBVA_MESSAGING_LEVEL", "0", true); + need_env_reset = true; + } + + // First try the default (ie., whatever $DISPLAY is set to). + unique_ptr va_dpy = try_open_va("", nullptr, nullptr); + if (va_dpy != nullptr) { + if (need_env_reset) { + unsetenv("LIBVA_MESSAGING_LEVEL"); + } + return ""; + } + + fprintf(stderr, "No --va-display was given, and the X11 display did not expose a VA-API H.264 encoder.\n"); + + // Try all /dev/dri/render* in turn. TODO: Accept /dev/dri/card*, too? + glob_t g; + int err = glob("/dev/dri/renderD*", 0, nullptr, &g); + if (err != 0) { + fprintf(stderr, "Couldn't list render nodes (%s) when trying to autodetect a replacement.\n", strerror(errno)); + } else { + for (size_t i = 0; i < g.gl_pathc; ++i) { + string path = g.gl_pathv[i]; + va_dpy = try_open_va(path, nullptr, nullptr); + if (va_dpy != nullptr) { + fprintf(stderr, "Autodetected %s as a suitable replacement; using it.\n", + path.c_str()); + globfree(&g); + if (need_env_reset) { + unsetenv("LIBVA_MESSAGING_LEVEL"); + } + return path; + } + } + } + + fprintf(stderr, "No suitable VA-API H.264 encoders were found in /dev/dri; giving up.\n"); + fprintf(stderr, "Note that if you are using an Intel CPU with an external GPU,\n"); + fprintf(stderr, "you may need to enable the integrated Intel GPU in your BIOS\n"); + fprintf(stderr, "to expose Quick Sync. Alternatively, you can use --record-x264-video\n"); + fprintf(stderr, "to use software instead of hardware H.264 encoding, at the expense\n"); + fprintf(stderr, "of increased CPU usage and possibly bit rate.\n"); + exit(1); +} diff --git a/nageru/quicksync_encoder.h b/nageru/quicksync_encoder.h new file mode 100644 index 0000000..110d615 --- /dev/null +++ b/nageru/quicksync_encoder.h @@ -0,0 +1,90 @@ +// Hardware H.264 encoding via VAAPI. Also orchestrates the H.264 encoding +// in general; this is unfortunate, and probably needs a cleanup. In particular, +// even if you don't actually use Quick Sync for anything, this class +// (or actually, QuickSyncEncoderImpl) still takes on a pretty central role. +// +// Heavily modified based on example code by Intel. Intel's original copyright +// and license is reproduced below: +// +// Copyright (c) 2007-2013 Intel Corporation. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sub license, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice (including the +// next paragraph) shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. +// IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR +// ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#ifndef _H264ENCODE_H +#define _H264ENCODE_H + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +#include "ref_counted_gl_sync.h" + +class DiskSpaceEstimator; +class Mux; +class QSurface; +class QuickSyncEncoderImpl; +class RefCountedFrame; +class X264Encoder; + +namespace movit { +class ResourcePool; +} // namespace movit + +// This is just a pimpl, because including anything X11-related in a .h file +// tends to trip up Qt. All the real logic is in QuickSyncEncoderImpl, +// defined in quicksync_encoder_impl.h. +// +// This class is _not_ thread-safe, except where mentioned. +class QuickSyncEncoder { +public: + QuickSyncEncoder(const std::string &filename, movit::ResourcePool *resource_pool, QSurface *surface, const std::string &va_display, int width, int height, AVOutputFormat *oformat, X264Encoder *x264_encoder, DiskSpaceEstimator *disk_space_estimator); + ~QuickSyncEncoder(); + + void set_stream_mux(Mux *mux); // Does not take ownership. Must be called unless x264 is used for the stream. + void add_audio(int64_t pts, std::vector audio); // Thread-safe. + bool is_zerocopy() const; // Thread-safe. + + // See VideoEncoder::begin_frame(). + bool begin_frame(int64_t pts, int64_t duration, movit::YCbCrLumaCoefficients ycbcr_coefficients, const std::vector &input_frames, GLuint *y_tex, GLuint *cbcr_tex); + RefCountedGLsync end_frame(); + void shutdown(); // Blocking. Does not require an OpenGL context. + void close_file(); // Does not require an OpenGL context. Must be run after shutdown. + void release_gl_resources(); // Requires an OpenGL context. Must be run after shutdown. + int64_t global_delay() const; // So we never get negative dts. + + // Tries to autodetect a device with a usable VA-API H.264 encoder. + // Tries first the default X11 display, then every /dev/dri/renderD* node in turn. + // Dies if none could be found. + static std::string get_usable_va_display(); + +private: + std::unique_ptr impl; +}; + +#endif diff --git a/nageru/quicksync_encoder_impl.h b/nageru/quicksync_encoder_impl.h new file mode 100644 index 0000000..7645b99 --- /dev/null +++ b/nageru/quicksync_encoder_impl.h @@ -0,0 +1,239 @@ +#ifndef _QUICKSYNC_ENCODER_IMPL_H +#define _QUICKSYNC_ENCODER_IMPL_H 1 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "audio_encoder.h" +#include "defs.h" +#include "timebase.h" +#include "print_latency.h" +#include "ref_counted_gl_sync.h" + +#define SURFACE_NUM 16 /* 16 surfaces for source YUV */ +#define MAX_NUM_REF1 16 // Seemingly a hardware-fixed value, not related to SURFACE_NUM +#define MAX_NUM_REF2 32 // Seemingly a hardware-fixed value, not related to SURFACE_NUM + +struct __bitstream { + unsigned int *buffer; + int bit_offset; + int max_size_in_dword; +}; +typedef struct __bitstream bitstream; + +namespace movit { +class ResourcePool; +} +class DiskSpaceEstimator; +class QSurface; +class X264Encoder; + +struct VADisplayWithCleanup { + ~VADisplayWithCleanup(); + + VADisplay va_dpy; + Display *x11_display = nullptr; + bool can_use_zerocopy = true; + int drm_fd = -1; +}; +std::unique_ptr va_open_display(const std::string &va_display); // Can return nullptr on failure. + +class QuickSyncEncoderImpl { +public: + QuickSyncEncoderImpl(const std::string &filename, movit::ResourcePool *resource_pool, QSurface *surface, const std::string &va_display, int width, int height, AVOutputFormat *oformat, X264Encoder *x264_encoder, DiskSpaceEstimator *disk_space_estimator); + ~QuickSyncEncoderImpl(); + void add_audio(int64_t pts, std::vector audio); + bool is_zerocopy() const; + bool begin_frame(int64_t pts, int64_t duration, movit::YCbCrLumaCoefficients ycbcr_coefficients, const std::vector &input_frames, GLuint *y_tex, GLuint *cbcr_tex); + RefCountedGLsync end_frame(); + void shutdown(); + void close_file(); + void release_gl_resources(); + void set_stream_mux(Mux *mux) + { + stream_mux = mux; + } + + // So we never get negative dts. + int64_t global_delay() const { + return int64_t(ip_period - 1) * (TIMEBASE / MAX_FPS); + } + +private: + struct storage_task { + unsigned long long display_order; + int frame_type; + std::vector audio; + int64_t pts, dts, duration; + movit::YCbCrLumaCoefficients ycbcr_coefficients; + ReceivedTimestamps received_ts; + std::vector ref_display_frame_numbers; + }; + struct PendingFrame { + RefCountedGLsync fence; + std::vector input_frames; + int64_t pts, duration; + movit::YCbCrLumaCoefficients ycbcr_coefficients; + }; + struct GLSurface { + // Only if x264_video_to_disk == false. + VASurfaceID src_surface, ref_surface; + VABufferID coded_buf; + VAImage surface_image; + + // Only if use_zerocopy == true (which implies x264_video_to_disk == false). + GLuint y_tex, cbcr_tex; + EGLImage y_egl_image, cbcr_egl_image; + + // Only if use_zerocopy == false. + GLuint pbo; + uint8_t *y_ptr, *cbcr_ptr; + size_t y_offset, cbcr_offset; + + // Surfaces can be busy (have refcount > 0) for a variety of + // reasons: First of all because they belong to a frame that's + // under encoding. But also reference frames take refcounts; + // while a frame is being encoded, all its reference frames + // also have increased refcounts so that they are not dropped. + // Similarly, just being in increases the + // refcount. Until it is back to zero, the surface cannot be given + // out for encoding another frame. Use release_gl_surface() + // to reduce the refcount, which will free the surface if + // the refcount reaches zero. + // + // Protected by storage_task_queue_mutex. + int refcount = 0; + }; + + void open_output_file(const std::string &filename); + void encode_thread_func(); + void encode_remaining_frames_as_p(int encoding_frame_num, int gop_start_display_frame_num, int64_t last_dts); + void add_packet_for_uncompressed_frame(int64_t pts, int64_t duration, const uint8_t *data); + void pass_frame(PendingFrame frame, int display_frame_num, int64_t pts, int64_t duration); + void encode_frame(PendingFrame frame, int encoding_frame_num, int display_frame_num, int gop_start_display_frame_num, + int frame_type, int64_t pts, int64_t dts, int64_t duration, movit::YCbCrLumaCoefficients ycbcr_coefficients); + void storage_task_thread(); + void storage_task_enqueue(storage_task task); + void save_codeddata(GLSurface *surf, storage_task task); + int render_packedsequence(movit::YCbCrLumaCoefficients ycbcr_coefficients); + int render_packedpicture(); + void render_packedslice(); + int render_sequence(); + int render_picture(GLSurface *surf, int frame_type, int display_frame_num, int gop_start_display_frame_num); + void sps_rbsp(movit::YCbCrLumaCoefficients ycbcr_coefficients, bitstream *bs); + void pps_rbsp(bitstream *bs); + int build_packed_pic_buffer(unsigned char **header_buffer); + int render_slice(int encoding_frame_num, int display_frame_num, int gop_start_display_frame_num, int frame_type); + void slice_header(bitstream *bs); + int build_packed_seq_buffer(movit::YCbCrLumaCoefficients ycbcr_coefficients, unsigned char **header_buffer); + int build_packed_slice_buffer(unsigned char **header_buffer); + int init_va(const std::string &va_display); + void enable_zerocopy_if_possible(); + int setup_encode(); + void release_encode(); + void update_ReferenceFrames(int current_display_frame, int frame_type); + void update_RefPicList_P(VAPictureH264 RefPicList0_P[MAX_NUM_REF2]); + void update_RefPicList_B(VAPictureH264 RefPicList0_B[MAX_NUM_REF2], VAPictureH264 RefPicList1_B[MAX_NUM_REF2]); + GLSurface *allocate_gl_surface(); + void release_gl_surface(size_t display_frame_num); + + bool is_shutdown = false; + bool has_released_gl_resources = false; + std::atomic use_zerocopy{false}; + + std::thread encode_thread, storage_thread; + + std::mutex storage_task_queue_mutex; + std::condition_variable storage_task_queue_changed; + std::queue storage_task_queue; // protected by storage_task_queue_mutex + bool storage_thread_should_quit = false; // protected by storage_task_queue_mutex + + std::mutex frame_queue_mutex; + std::condition_variable frame_queue_nonempty; + bool encode_thread_should_quit = false; // under frame_queue_mutex + + int current_storage_frame; + + PendingFrame current_video_frame; // Used only between begin_frame() and end_frame(). + std::queue pending_video_frames; // under frame_queue_mutex + movit::ResourcePool *resource_pool; + QSurface *surface; + + // Frames that are done rendering and passed on to x264 (if enabled), + // but have not been encoded by Quick Sync yet, and thus also not freed. + // The key is the display frame number. + std::map reorder_buffer; + int quicksync_encoding_frame_num = 0; + + std::mutex file_audio_encoder_mutex; + std::unique_ptr file_audio_encoder; + + X264Encoder *x264_encoder; // nullptr if not using x264. + + Mux* stream_mux = nullptr; // To HTTP. + std::unique_ptr file_mux; // To local disk. + + // Encoder parameters + std::unique_ptr va_dpy; + VAProfile h264_profile = (VAProfile)~0; + VAConfigAttrib config_attrib[VAConfigAttribTypeMax]; + int config_attrib_num = 0, enc_packed_header_idx; + + GLSurface gl_surfaces[SURFACE_NUM]; + + // For all frames in encoding (refcount > 0), a pointer into gl_surfaces + // for the surface used for that frame. Protected by storage_task_queue_mutex. + // The key is display frame number. + std::unordered_map surface_for_frame; + + VAConfigID config_id; + VAContextID context_id; + VAEncSequenceParameterBufferH264 seq_param; + VAEncPictureParameterBufferH264 pic_param; + VAEncSliceParameterBufferH264 slice_param; + VAPictureH264 CurrentCurrPic; + + struct ReferenceFrame { + VAPictureH264 pic; + int display_number; // To track reference counts. + }; + std::deque reference_frames; + + // Static quality settings. + static constexpr unsigned int frame_bitrate = 15000000 / 60; // Doesn't really matter; only initial_qp does. + static constexpr unsigned int num_ref_frames = 2; + static constexpr int initial_qp = 15; + static constexpr int minimal_qp = 0; + static constexpr int intra_period = 30; + static constexpr int intra_idr_period = MAX_FPS; // About a second; more at lower frame rates. Not ideal. + + // Quality settings that are meant to be static, but might be overridden + // by the profile. + int constraint_set_flag = 0; + int h264_packedheader = 0; /* support pack header? */ + int h264_maxref = (1<<16|1); + int h264_entropy_mode = 1; /* cabac */ + int ip_period = 3; + + unsigned int current_ref_frame_num = 0; // Encoding frame order within this GOP, sans B-frames. + + int frame_width; + int frame_height; + int frame_width_mbaligned; + int frame_height_mbaligned; + + DiskSpaceEstimator *disk_space_estimator; +}; + +#endif // !defined(_QUICKSYNC_ENCODER_IMPL_H) diff --git a/nageru/quittable_sleeper.h b/nageru/quittable_sleeper.h new file mode 100644 index 0000000..6c449a7 --- /dev/null +++ b/nageru/quittable_sleeper.h @@ -0,0 +1,74 @@ +#ifndef _QUITTABLE_SLEEPER +#define _QUITTABLE_SLEEPER 1 + +// A class that assists with fast shutdown of threads. You can set +// a flag that says the thread should quit, which it can then check +// in a loop -- and if the thread sleeps (using the sleep_* functions +// on the class), that sleep will immediately be aborted. +// +// All member functions on this class are thread-safe. + +#include +#include +#include + +class QuittableSleeper { +public: + void quit() + { + std::lock_guard l(mu); + should_quit_var = true; + quit_cond.notify_all(); + } + + void unquit() + { + std::lock_guard l(mu); + should_quit_var = false; + } + + void wakeup() + { + std::lock_guard l(mu); + should_wakeup_var = true; + quit_cond.notify_all(); + } + + bool should_quit() const + { + std::lock_guard l(mu); + return should_quit_var; + } + + // Returns false if woken up early. + template + bool sleep_for(const std::chrono::duration &duration) + { + std::chrono::steady_clock::time_point t = + std::chrono::steady_clock::now() + + std::chrono::duration_cast(duration); + return sleep_until(t); + } + + // Returns false if woken up early. + template + bool sleep_until(const std::chrono::time_point &t) + { + std::unique_lock lock(mu); + quit_cond.wait_until(lock, t, [this]{ + return should_quit_var || should_wakeup_var; + }); + if (should_wakeup_var) { + should_wakeup_var = false; + return false; + } + return !should_quit_var; + } + +private: + mutable std::mutex mu; + bool should_quit_var = false, should_wakeup_var = false; + std::condition_variable quit_cond; +}; + +#endif // !defined(_QUITTABLE_SLEEPER) diff --git a/nageru/ref_counted_frame.cpp b/nageru/ref_counted_frame.cpp new file mode 100644 index 0000000..0799017 --- /dev/null +++ b/nageru/ref_counted_frame.cpp @@ -0,0 +1,11 @@ +#include "ref_counted_frame.h" + +#include + +void release_refcounted_frame(bmusb::FrameAllocator::Frame *frame) +{ + if (frame->owner) { + frame->owner->release_frame(*frame); + } + delete frame; +} diff --git a/nageru/ref_counted_frame.h b/nageru/ref_counted_frame.h new file mode 100644 index 0000000..59a1686 --- /dev/null +++ b/nageru/ref_counted_frame.h @@ -0,0 +1,60 @@ +#ifndef _REF_COUNTED_FRAME_H +#define _REF_COUNTED_FRAME_H 1 + +// A wrapper around FrameAllocator::Frame that is automatically refcounted; +// when the refcount goes to zero, the frame is given back to the allocator. +// +// Note that the important point isn't really the pointer to the Frame itself, +// it's the resources it's representing that need to go back to the allocator. +// +// FIXME: There's an issue here in that we could be releasing a frame while +// we're still uploading textures from it, causing it to be written to in +// another thread. (Thankfully, it goes to the back of the queue, and there's +// usually a render in-between, meaning it's fairly unlikely that someone +// actually managed to get to that race.) We should probably have some mechanism +// for registering fences. + +#include + +#include "bmusb/bmusb.h" + +void release_refcounted_frame(bmusb::FrameAllocator::Frame *frame); + +typedef std::shared_ptr RefCountedFrameBase; + +class RefCountedFrame : public RefCountedFrameBase { +public: + RefCountedFrame() {} + + RefCountedFrame(const bmusb::FrameAllocator::Frame &frame) + : RefCountedFrameBase(new bmusb::FrameAllocator::Frame(frame), release_refcounted_frame) {} +}; + +// Similar to RefCountedFrame, but as unique_ptr instead of shared_ptr. + +struct Unique_frame_deleter { + void operator() (bmusb::FrameAllocator::Frame *frame) const { + release_refcounted_frame(frame); + } +}; + +typedef std::unique_ptr + UniqueFrameBase; + +class UniqueFrame : public UniqueFrameBase { +public: + UniqueFrame() {} + + UniqueFrame(const bmusb::FrameAllocator::Frame &frame) + : UniqueFrameBase(new bmusb::FrameAllocator::Frame(frame)) {} + + bmusb::FrameAllocator::Frame get_and_release() + { + bmusb::FrameAllocator::Frame *ptr = release(); + bmusb::FrameAllocator::Frame frame = *ptr; + delete ptr; + return frame; + } +}; + +#endif // !defined(_REF_COUNTED_FRAME_H) diff --git a/nageru/ref_counted_gl_sync.h b/nageru/ref_counted_gl_sync.h new file mode 100644 index 0000000..8b6db68 --- /dev/null +++ b/nageru/ref_counted_gl_sync.h @@ -0,0 +1,39 @@ +#ifndef _REF_COUNTED_GL_SYNC_H +#define _REF_COUNTED_GL_SYNC_H 1 + +// A wrapper around GLsync (OpenGL fences) that is automatically refcounted. +// Useful since we sometimes want to use the same fence two entirely different +// places. (We could set two fences at the same time, but they are not an +// unlimited hardware resource, so it would be a bit wasteful.) + +#include +#include +#include + +typedef std::shared_ptr<__GLsync> RefCountedGLsyncBase; + +class RefCountedGLsync : public RefCountedGLsyncBase { +public: + RefCountedGLsync() {} + + RefCountedGLsync(GLenum condition, GLbitfield flags) + : RefCountedGLsyncBase(locked_glFenceSync(condition, flags), glDeleteSync) {} + +private: + // These are to work around apitrace bug #446. + static GLsync locked_glFenceSync(GLenum condition, GLbitfield flags) + { + std::lock_guard lock(fence_lock); + return glFenceSync(condition, flags); + } + + static void locked_glDeleteSync(GLsync sync) + { + std::lock_guard lock(fence_lock); + glDeleteSync(sync); + } + + static std::mutex fence_lock; +}; + +#endif // !defined(_REF_COUNTED_GL_SYNC_H) diff --git a/nageru/resampling_queue.cpp b/nageru/resampling_queue.cpp new file mode 100644 index 0000000..ef7b735 --- /dev/null +++ b/nageru/resampling_queue.cpp @@ -0,0 +1,200 @@ +// Parts of the code is adapted from Adriaensen's project Zita-ajbridge +// (as of November 2015), although it has been heavily reworked for this use +// case. Original copyright follows: +// +// Copyright (C) 2012-2015 Fons Adriaensen +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include "resampling_queue.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace std::chrono; + +ResamplingQueue::ResamplingQueue(DeviceSpec device_spec, unsigned freq_in, unsigned freq_out, unsigned num_channels, double expected_delay_seconds) + : device_spec(device_spec), freq_in(freq_in), freq_out(freq_out), num_channels(num_channels), + current_estimated_freq_in(freq_in), + ratio(double(freq_out) / double(freq_in)), expected_delay(expected_delay_seconds * OUTPUT_FREQUENCY) +{ + vresampler.setup(ratio, num_channels, /*hlen=*/32); + + // Prime the resampler so there's no more delay. + vresampler.inp_count = vresampler.inpsize() / 2 - 1; + vresampler.out_count = 1048576; + vresampler.process (); +} + +void ResamplingQueue::add_input_samples(steady_clock::time_point ts, const float *samples, ssize_t num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy) +{ + if (num_samples == 0) { + return; + } + + assert(duration(ts.time_since_epoch()).count() >= 0.0); + + bool good_sample = (rate_adjustment_policy == ADJUST_RATE); + if (good_sample && a1.good_sample) { + a0 = a1; + } + a1.ts = ts; + a1.input_samples_received += num_samples; + a1.good_sample = good_sample; + if (a0.good_sample && a1.good_sample) { + current_estimated_freq_in = (a1.input_samples_received - a0.input_samples_received) / duration(a1.ts - a0.ts).count(); + if (!(current_estimated_freq_in >= 0.0)) { + fprintf(stderr, "%s: PANIC: Input audio clock going backwards, ignoring.\n", + spec_to_string(device_spec).c_str()); + current_estimated_freq_in = freq_in; + } + + // Bound the frequency, so that a single wild result won't throw the filter off guard. + current_estimated_freq_in = min(current_estimated_freq_in, 1.2 * freq_in); + current_estimated_freq_in = max(current_estimated_freq_in, 0.8 * freq_in); + } + + buffer.insert(buffer.end(), samples, samples + num_samples * num_channels); +} + +bool ResamplingQueue::get_output_samples(steady_clock::time_point ts, float *samples, ssize_t num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy) +{ + assert(num_samples > 0); + if (a1.input_samples_received == 0) { + // No data yet, just return zeros. + memset(samples, 0, num_samples * num_channels * sizeof(float)); + return true; + } + + // This can happen when we get dropped frames on the master card. + if (duration(ts.time_since_epoch()).count() <= 0.0) { + rate_adjustment_policy = DO_NOT_ADJUST_RATE; + } + + if (rate_adjustment_policy == ADJUST_RATE && (a0.good_sample || a1.good_sample)) { + // Estimate the current number of input samples produced at + // this instant in time, by extrapolating from the last known + // good point. Note that we could be extrapolating backward or + // forward, depending on the timing of the calls. + const InputPoint &base_point = a1.good_sample ? a1 : a0; + assert(duration(base_point.ts.time_since_epoch()).count() >= 0.0); + + // NOTE: Due to extrapolation, input_samples_received can + // actually go negative here the few first calls (ie., we asked + // about a timestamp where we hadn't actually started producing + // samples yet), but that is harmless. + const double input_samples_received = base_point.input_samples_received + + current_estimated_freq_in * duration(ts - base_point.ts).count(); + + // Estimate the number of input samples _consumed_ after we've run the resampler. + const double input_samples_consumed = total_consumed_samples + + num_samples / (ratio * rcorr); + + double actual_delay = input_samples_received - input_samples_consumed; + actual_delay += vresampler.inpdist(); // Delay in the resampler itself. + double err = actual_delay - expected_delay; + if (first_output) { + // Before the very first block, insert artificial delay based on our initial estimate, + // so that we don't need a long period to stabilize at the beginning. + if (err < 0.0) { + int delay_samples_to_add = lrintf(-err); + for (ssize_t i = 0; i < delay_samples_to_add * num_channels; ++i) { + buffer.push_front(0.0f); + } + total_consumed_samples -= delay_samples_to_add; // Equivalent to increasing input_samples_received on a0 and a1. + err += delay_samples_to_add; + } else if (err > 0.0) { + int delay_samples_to_remove = min(lrintf(err), buffer.size() / num_channels); + buffer.erase(buffer.begin(), buffer.begin() + delay_samples_to_remove * num_channels); + total_consumed_samples += delay_samples_to_remove; + err -= delay_samples_to_remove; + } + } + first_output = false; + + // Compute loop filter coefficients for the two filters. We need to compute them + // every time, since they depend on the number of samples the user asked for. + // + // The loop bandwidth is at 0.02 Hz; our jitter is pretty large + // since none of the threads involved run at real-time priority. + // However, the first four seconds, we use a larger loop bandwidth (2 Hz), + // because there's a lot going on during startup, and thus the + // initial estimate might be tainted by jitter during that phase, + // and we want to converge faster. + // + // NOTE: The above logic might only hold during Nageru startup + // (we start ResamplingQueues also when we e.g. switch sound sources), + // but in general, a little bit of increased timing jitter is acceptable + // right after a setup change like this. + double loop_bandwidth_hz = (total_consumed_samples < 4 * freq_in) ? 0.2 : 0.02; + + // Set filters. The first filter much wider than the first one (20x as wide). + double w = (2.0 * M_PI) * loop_bandwidth_hz * num_samples / freq_out; + double w0 = 1.0 - exp(-20.0 * w); + double w1 = w * 1.5 / num_samples / ratio; + double w2 = w / 1.5; + + // Filter through the loop filter to find the correction ratio. + z1 += w0 * (w1 * err - z1); + z2 += w0 * (z1 - z2); + z3 += w2 * z2; + rcorr = 1.0 - z2 - z3; + if (rcorr > 1.05) rcorr = 1.05; + if (rcorr < 0.95) rcorr = 0.95; + assert(!isnan(rcorr)); + vresampler.set_rratio(rcorr); + } + + // Finally actually resample, producing exactly output samples. + vresampler.out_data = samples; + vresampler.out_count = num_samples; + while (vresampler.out_count > 0) { + if (buffer.empty()) { + // This should never happen unless delay is set way too low, + // or we're dropping a lot of data. + fprintf(stderr, "%s: PANIC: Out of input samples to resample, still need %d output samples! (correction factor is %f)\n", + spec_to_string(device_spec).c_str(), int(vresampler.out_count), rcorr); + memset(vresampler.out_data, 0, vresampler.out_count * num_channels * sizeof(float)); + + // Reset the loop filter. + z1 = z2 = z3 = 0.0; + + return false; + } + + float inbuf[1024]; + size_t num_input_samples = sizeof(inbuf) / (sizeof(float) * num_channels); + if (num_input_samples * num_channels > buffer.size()) { + num_input_samples = buffer.size() / num_channels; + } + copy(buffer.begin(), buffer.begin() + num_input_samples * num_channels, inbuf); + + vresampler.inp_count = num_input_samples; + vresampler.inp_data = inbuf; + + int err = vresampler.process(); + assert(err == 0); + + size_t consumed_samples = num_input_samples - vresampler.inp_count; + total_consumed_samples += consumed_samples; + buffer.erase(buffer.begin(), buffer.begin() + consumed_samples * num_channels); + } + return true; +} diff --git a/nageru/resampling_queue.h b/nageru/resampling_queue.h new file mode 100644 index 0000000..f0e2499 --- /dev/null +++ b/nageru/resampling_queue.h @@ -0,0 +1,118 @@ +#ifndef _RESAMPLING_QUEUE_H +#define _RESAMPLING_QUEUE_H 1 + +// Takes in samples from an input source, possibly with jitter, and outputs a fixed number +// of samples every iteration. Used to a) change sample rates if needed, and b) deal with +// input sources that don't have audio locked to video. For every input video +// frame, you call add_input_samples() with the received time point of the video frame, +// taken to be the _end_ point of the frame's audio. When you want to _output_ a finished +// frame with audio, you get_output_samples() with the number of samples you want, and will +// get exactly that number of samples back. If the input and output clocks are not in sync, +// the audio will be stretched for you. (If they are _very_ out of sync, this will come through +// as a pitch shift.) Of course, the process introduces some delay; you specify a target delay +// (typically measured in milliseconds, although more is fine) and the algorithm works to +// provide exactly that. +// +// A/V sync is a much harder problem than one would intuitively assume. This implementation +// is based on a 2012 paper by Fons Adriaensen, “Controlling adaptive resampling” +// (http://kokkinizita.linuxaudio.org/papers/adapt-resamp.pdf). The paper gives an algorithm +// that converges to jitter of <100 ns; the basic idea is to measure the _rate_ the input +// queue fills and is drained (as opposed to the length of the queue itself), and smoothly +// adjust the resampling rate so that it reaches steady state at the desired delay. +// +// Parts of the code is adapted from Adriaensen's project Zita-ajbridge (based on the same +// algorithm), although it has been heavily reworked for this use case. Original copyright follows: +// +// Copyright (C) 2012-2015 Fons Adriaensen +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include +#include +#include +#include +#include + +#include "defs.h" +#include "input_mapping.h" + +class ResamplingQueue { +public: + // device_spec is for debugging outputs only. + ResamplingQueue(DeviceSpec device_spec, unsigned freq_in, unsigned freq_out, unsigned num_channels, double expected_delay_seconds); + + // If policy is DO_NOT_ADJUST_RATE, the resampling rate will not be changed. + // This is primarily useful if you have an extraordinary situation, such as + // dropped frames. + enum RateAdjustmentPolicy { + DO_NOT_ADJUST_RATE, + ADJUST_RATE + }; + + void add_input_samples(std::chrono::steady_clock::time_point ts, const float *samples, ssize_t num_samples, RateAdjustmentPolicy rate_adjustment_policy); + // Returns false if underrun. + bool get_output_samples(std::chrono::steady_clock::time_point ts, float *samples, ssize_t num_samples, RateAdjustmentPolicy rate_adjustment_policy); + +private: + void init_loop_filter(double bandwidth_hz); + + VResampler vresampler; + + DeviceSpec device_spec; + unsigned freq_in, freq_out, num_channels; + + bool first_output = true; + + struct InputPoint { + // Equivalent to t_a0 or t_a1 in the paper. + std::chrono::steady_clock::time_point ts; + + // Number of samples that have been written to the queue (in total) + // at this time point. Equivalent to k_a0 or k_a1 in the paper. + size_t input_samples_received = 0; + + // Set to false if we should not use the timestamp from this sample + // (e.g. if it is from a dropped frame and thus bad). In particular, + // we will not use it for updateing current_estimated_freq_in. + bool good_sample = false; + }; + InputPoint a0, a1; + + // The current rate at which we seem to get input samples, in Hz. + // For an ideal input, identical to freq_in. + double current_estimated_freq_in; + + ssize_t total_consumed_samples = 0; + + // Filter state for the loop filter. + double z1 = 0.0, z2 = 0.0, z3 = 0.0; + + // Ratio between the two frequencies. + const double ratio; + + // Current correction ratio. ratio * rcorr gives the true ratio, + // so values above 1.0 means to pitch down (consume input samples slower). + double rcorr = 1.0; + + // How much delay we are expected to have, in input samples. + // If actual delay drifts too much away from this, we will start + // changing the resampling ratio to compensate. + const double expected_delay; + + // Input samples not yet fed into the resampler. + // TODO: Use a circular buffer instead, for efficiency. + std::deque buffer; +}; + +#endif // !defined(_RESAMPLING_QUEUE_H) diff --git a/nageru/scripts/compile_cef_dll_wrapper.sh b/nageru/scripts/compile_cef_dll_wrapper.sh new file mode 100755 index 0000000..ee8dbb2 --- /dev/null +++ b/nageru/scripts/compile_cef_dll_wrapper.sh @@ -0,0 +1,14 @@ +#! /bin/sh +set -e + +BUILD_DIR="$1" +CEF_DIR="$2" +CMAKE="$3" +OUTPUT="$4" +STAMP="$5" + +! [ -d "$BUILD_DIR" ] || rm -r "$BUILD_DIR" +mkdir "$BUILD_DIR" +( cd "$BUILD_DIR" && $CMAKE -G Ninja "$CEF_DIR" && ninja libcef_dll_wrapper ) +cp "$BUILD_DIR"/libcef_dll_wrapper/libcef_dll_wrapper.a "$OUTPUT" +touch "$STAMP" diff --git a/nageru/scripts/setup_nageru_symlink.sh b/nageru/scripts/setup_nageru_symlink.sh new file mode 100755 index 0000000..296f5cf --- /dev/null +++ b/nageru/scripts/setup_nageru_symlink.sh @@ -0,0 +1,3 @@ +#! /bin/sh +set -e +ln -sf ${MESON_INSTALL_PREFIX}/lib/nageru/nageru ${MESON_INSTALL_DESTDIR_PREFIX}/bin/nageru diff --git a/nageru/simple.lua b/nageru/simple.lua new file mode 100644 index 0000000..2bff8f3 --- /dev/null +++ b/nageru/simple.lua @@ -0,0 +1,179 @@ +-- The theme is what decides what's actually shown on screen, what kind of +-- transitions are available (if any), and what kind of inputs there are, +-- if any. In general, it drives the entire display logic by creating Movit +-- chains, setting their parameters and then deciding which to show when. +-- +-- Themes are written in Lua, which reflects a simplified form of the Movit API +-- where all the low-level details (such as texture formats) are handled by the +-- C++ side and you generally just build chains. +-- +-- This is a much simpler theme than the default theme; it only allows you to +-- switch between inputs and set white balance, no transitions or the likes. +-- Thus, it should be simpler to understand. + +local input_neutral_color = {{0.5, 0.5, 0.5}, {0.5, 0.5, 0.5}} + +local live_signal_num = 0 +local preview_signal_num = 1 + +-- A chain to show a single input, with white balance. In a real example, +-- we'd probably want to support deinterlacing and high-quality scaling +-- (if the input isn't exactly what we want). However, we don't want these +-- things always on, so we'd need to generate more chains for the various +-- cases. In such a simple example, just having two is fine. +function make_simple_chain(hq) + local chain = EffectChain.new(16, 9) + + local input = chain:add_live_input(false, false) -- No deinterlacing, no bounce override. + input:connect_signal(0) -- First input card. Can be changed whenever you want. + local wb_effect = chain:add_effect(WhiteBalanceEffect.new()) + chain:finalize(hq) + + return { + chain = chain, + input = input, + wb_effect = wb_effect, + } +end + +-- We only make two chains; one for the live view and one for the previews. +-- (Since they have different outputs, you cannot mix and match them.) +local simple_hq_chain = make_simple_chain(true) +local simple_lq_chain = make_simple_chain(false) + +-- API ENTRY POINT +-- Returns the number of outputs in addition to the live (0) and preview (1). +-- Called only once, at the start of the program. +function num_channels() + return 2 +end + +-- API ENTRY POINT +-- Returns the name for each additional channel (starting from 2). +-- Called at the start of the program, and then each frame for live +-- channels in case they change resolution. +function channel_name(channel) + if channel == 2 then + return "First input" + elseif channel == 3 then + return "Second input" + end +end + +-- API ENTRY POINT +-- Returns, given a channel number, which signal it corresponds to (starting from 0). +-- Should return -1 if the channel does not correspond to a simple signal. +-- (The information is used for whether right-click on the channel should bring up +-- an input selector or not.) +-- Called once for each channel, at the start of the program. +-- Will never be called for live (0) or preview (1). +function channel_signal(channel) + if channel == 2 then + return 0 + elseif channel == 3 then + return 1 + else + return -1 + end +end + +-- API ENTRY POINT +-- Called every frame. Returns the color (if any) to paint around the given +-- channel. Returns a CSS color (typically to mark live and preview signals); +-- "transparent" is allowed. +-- Will never be called for live (0) or preview (1). +function channel_color(channel) + return "transparent" +end + +-- API ENTRY POINT +-- Returns if a given channel supports setting white balance (starting from 2). +-- Called only once for each channel, at the start of the program. +function supports_set_wb(channel) + return channel == 2 or channel == 3 +end + +-- API ENTRY POINT +-- Gets called with a new gray point when the white balance is changing. +-- The color is in linear light (not sRGB gamma). +function set_wb(channel, red, green, blue) + if channel == 2 then + input_neutral_color[1] = { red, green, blue } + elseif channel == 3 then + input_neutral_color[2] = { red, green, blue } + end +end + +-- API ENTRY POINT +-- Called every frame. +function get_transitions(t) + if live_signal_num == preview_signal_num then + -- No transitions possible. + return {} + else + return {"Cut"} + end +end + +-- API ENTRY POINT +-- Called when the user clicks a transition button. For our case, +-- we only do cuts, so we ignore the parameters; just switch live and preview. +function transition_clicked(num, t) + local temp = live_signal_num + live_signal_num = preview_signal_num + preview_signal_num = temp +end + +-- API ENTRY POINT +function channel_clicked(num) + preview_signal_num = num +end + +-- API ENTRY POINT +-- Called every frame. Get the chain for displaying at input , +-- where 0 is live, 1 is preview, 2 is the first channel to display +-- in the bottom bar, and so on up to num_channels()+1. t is the +-- current time in seconds. width and height are the dimensions of +-- the output, although you can ignore them if you don't need them +-- (they're useful if you want to e.g. know what to resample by). +-- +-- is basically an exposed InputState, which you can use to +-- query for information about the signals at the point of the current +-- frame. In particular, you can call get_width() and get_height() +-- for any signal number, and use that to e.g. assist in chain selection. +-- +-- You should return two objects; the chain itself, and then a +-- function (taking no parameters) that is run just before rendering. +-- The function needs to call connect_signal on any inputs, so that +-- it gets updated video data for the given frame. (You are allowed +-- to switch which input your input is getting from between frames, +-- but not calling connect_signal results in undefined behavior.) +-- If you want to change any parameters in the chain, this is also +-- the right place. +-- +-- NOTE: The chain returned must be finalized with the Y'CbCr flag +-- if and only if num==0. +function get_chain(num, t, width, height, signals) + local chain, signal_num + if num == 0 then -- Live (right pane). + chain = simple_hq_chain + signal_num = live_signal_num + elseif num == 1 then -- Preview (left pane). + chain = simple_lq_chain + signal_num = preview_signal_num + else -- One of the two previews (bottom panes). + chain = simple_lq_chain + signal_num = num - 2 + end + + -- Make a copy of the current neutral color before returning, so that the + -- returned prepare function is unaffected by state changes made by the UI + -- before it is rendered. + local color = input_neutral_color[signal_num + 1] + + local prepare = function() + chain.input:connect_signal(signal_num) + chain.wb_effect:set_vec3("neutral_color", color[1], color[2], color[3]) + end + return chain.chain, prepare +end diff --git a/nageru/state.proto b/nageru/state.proto new file mode 100644 index 0000000..6372e61 --- /dev/null +++ b/nageru/state.proto @@ -0,0 +1,35 @@ +// Used to serialize state between runs. Currently only audio input mappings, +// but in theory we could do the entire mix, video inputs, etc. + +syntax = "proto2"; + +// Similar to DeviceSpec, but only devices that are used are stored, +// and contains additional information that will help us try to map +// to the right device even if the devices have moved around. +message DeviceSpecProto { + // Members from DeviceSpec itself. + enum InputSourceType { SILENCE = 0; CAPTURE_CARD = 1; ALSA_INPUT = 2; FFMPEG_VIDEO_INPUT = 3; }; + optional InputSourceType type = 1; + optional int32 index = 2; + + // Additional information. + optional string display_name = 3; + optional string alsa_name = 4; // Only for ALSA devices. + optional string alsa_info = 5; // Only for ALSA devices. + optional int32 num_channels = 6; // Only for ALSA devices. + optional string address = 7; // Only for ALSA devices. +} + +// Corresponds to InputMapping::Bus. +message BusProto { + optional string name = 1; + optional int32 device_index = 2; // Index into the "devices" array. + optional int32 source_channel_left = 3; + optional int32 source_channel_right = 4; +} + +// Corresponds to InputMapping. +message InputMappingProto { + repeated DeviceSpecProto device = 1; + repeated BusProto bus = 2; +} diff --git a/nageru/stereocompressor.cpp b/nageru/stereocompressor.cpp new file mode 100644 index 0000000..d9f1142 --- /dev/null +++ b/nageru/stereocompressor.cpp @@ -0,0 +1,137 @@ +#include "stereocompressor.h" + +#include +#include +#include + +using namespace std; + +namespace { + +// Implement a less accurate but faster pow(x, y). We use the standard identity +// +// x^y = exp(y * ln(x)) +// +// with the ranges: +// +// x in 1..(1/threshold) +// y in -1..0 +// +// Assume threshold goes from 0 to -40 dB. That means 1/threshold = 100, +// so input to ln(x) can be 1..100. Worst case for end accuracy is y=-1. +// To get a good minimax approximation (not the least wrt. continuity +// at x=1), I had to make a piecewise linear function for the two ranges: +// +// with(numapprox): +// f1 := minimax(ln, 1..6, [3, 3], x -> 1/x, 'maxerror'); +// f2 := minimax(ln, 6..100, [3, 3], x -> 1/x, 'maxerror'); +// f := x -> piecewise(x < 6, f1(x), f2(x)); +// +// (Continuity: Error is down to the 1e-6 range for x=1, difference between +// f1 and f2 range at the crossover point is in the 1e-5 range. The cutoff +// point at x=6 is chosen to get maxerror pretty close between f1 and f2.) +// +// Maximum output of ln(x) here is of course ln(100) ~= 4.605. So we can find +// an approximation for exp over the range -4.605..0, where we care mostly +// about the relative error: +// +// g := minimax(exp, -ln(100)..0, [3, 3], x -> 1/exp(x), 'maxerror'); +// +// We can find the worst-case error in dB from this through a simple plot: +// +// dbdiff := (x, y) -> abs(20 * log10(x / y)); +// plot(dbdiff(g(-f(x)), 1/x), x=1..100); +// +// which readily shows the error never to be above ~0.001 dB or so +// (actually 0.00119 dB, for the case of x=100). y=-1 remains the worst case, +// it would seem. +// +// If we cared even more about speed, we could probably fuse y into +// the coefficients for ln_nom and postgain into the coefficients for ln_den. +// But if so, we should probably rather just SIMD the entire thing instead. +inline float fastpow(float x, float y) +{ + float ln_nom, ln_den; + if (x < 6.0f) { + ln_nom = -0.059237648f + (-0.0165117771f + (0.06818859075f + 0.007560968243f * x) * x) * x; + ln_den = 0.0202509098f + (0.08419174188f + (0.03647189417f + 0.001642577975f * x) * x) * x; + } else { + ln_nom = -0.005430534f + (0.00633589178f + (0.0006319155549f + 0.4789541675e-5f * x) * x) * x; + ln_den = 0.0064785099f + (0.003219629109f + (0.0001531823694f + 0.6884656640e-6f * x) * x) * x; + } + float v = y * ln_nom / ln_den; + float exp_nom = 0.2195097621f + (0.08546059868f + (0.01208501759f + 0.0006173448113f * v) * v) * v; + float exp_den = 0.2194980791f + (-0.1343051968f + (0.03556072737f - 0.006174398513f * v) * v) * v; + return exp_nom / exp_den; +} + +inline float compressor_knee(float x, float threshold, float inv_threshold, float inv_ratio_minus_one, float postgain) +{ + assert(inv_ratio_minus_one <= 0.0f); + if (x > threshold) { + return postgain * fastpow(x * inv_threshold, inv_ratio_minus_one); + } else { + return postgain; + } +} + +} // namespace + +void StereoCompressor::process(float *buf, size_t num_samples, float threshold, float ratio, + float attack_time, float release_time, float makeup_gain) +{ + float attack_increment = float(pow(2.0f, 1.0f / (attack_time * sample_rate + 1))); + if (attack_time == 0.0f) attack_increment = 100000; // For instant attack reaction. + + const float release_increment = float(pow(2.0f, -1.0f / (release_time * sample_rate + 1))); + const float peak_increment = float(pow(2.0f, -1.0f / (0.003f * sample_rate + 1))); + + float inv_ratio_minus_one = 1.0f / ratio - 1.0f; + if (ratio > 63) inv_ratio_minus_one = -1.0f; // Infinite ratio. + float inv_threshold = 1.0f / threshold; + + float *left_ptr = buf; + float *right_ptr = buf + 1; + + if (inv_ratio_minus_one >= 0.0) { + for (size_t i = 0; i < num_samples; ++i) { + *left_ptr *= makeup_gain; + left_ptr += 2; + + *right_ptr *= makeup_gain; + right_ptr += 2; + } + return; + } + + float peak_level = this->peak_level; + float compr_level = this->compr_level; + + for (size_t i = 0; i < num_samples; ++i) { + if (fabs(*left_ptr) > peak_level) peak_level = float(fabs(*left_ptr)); + if (fabs(*right_ptr) > peak_level) peak_level = float(fabs(*right_ptr)); + + if (peak_level > compr_level) { + compr_level = min(compr_level * attack_increment, peak_level); + } else { + compr_level = max(compr_level * release_increment, 0.0001f); + } + + float scalefactor_with_gain = compressor_knee(compr_level, threshold, inv_threshold, inv_ratio_minus_one, makeup_gain); + + *left_ptr *= scalefactor_with_gain; + left_ptr += 2; + + *right_ptr *= scalefactor_with_gain; + right_ptr += 2; + + peak_level = max(peak_level * peak_increment, 0.0001f); + } + + // Store attenuation level for debug/visualization. + scalefactor = compressor_knee(compr_level, threshold, inv_threshold, inv_ratio_minus_one, 1.0f); + + this->peak_level = peak_level; + this->compr_level = compr_level; +} + diff --git a/nageru/stereocompressor.h b/nageru/stereocompressor.h new file mode 100644 index 0000000..be13ce2 --- /dev/null +++ b/nageru/stereocompressor.h @@ -0,0 +1,44 @@ +#ifndef _STEREOCOMPRESSOR_H +#define _STEREOCOMPRESSOR_H 1 + +#include +// A simple compressor based on absolute values, with independent +// attack/release times. There is no sidechain or lookahead, but the +// peak value is shared between both channels. +// +// The compressor was originally written by, and is copyrighted by, Rune Holm. +// It has been adapted and relicensed under GPLv3 (or, at your option, +// any later version) for Nageru, so that its license matches the rest of the code. + +class StereoCompressor { +public: + StereoCompressor(float sample_rate) + : sample_rate(sample_rate) { + reset(); + } + + void reset() { + peak_level = compr_level = 0.1f; + scalefactor = 0.0f; + } + + // Process interleaved stereo data in-place. + // Attack and release times are in seconds. + void process(float *buf, size_t num_samples, float threshold, float ratio, + float attack_time, float release_time, float makeup_gain); + + // Last level estimated (after attack/decay applied). + float get_level() { return compr_level; } + + // Last attenuation factor applied, e.g. if 5x compression is currently applied, + // this number will be 0.2. + float get_attenuation() { return scalefactor; } + +private: + float sample_rate; + float peak_level; + float compr_level; + float scalefactor; +}; + +#endif /* !defined(_STEREOCOMPRESSOR_H) */ diff --git a/nageru/theme.cpp b/nageru/theme.cpp new file mode 100644 index 0000000..ca4de4f --- /dev/null +++ b/nageru/theme.cpp @@ -0,0 +1,1577 @@ +#include "theme.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "defs.h" +#ifdef HAVE_CEF +#include "cef_capture.h" +#endif +#include "ffmpeg_capture.h" +#include "flags.h" +#include "image_input.h" +#include "input_state.h" +#include "pbo_frame_allocator.h" + +#if !defined LUA_VERSION_NUM || LUA_VERSION_NUM==501 + +// Compatibility shims for LuaJIT 2.0 (LuaJIT 2.1 implements the entire Lua 5.2 API). +// Adapted from https://github.com/keplerproject/lua-compat-5.2/blob/master/c-api/compat-5.2.c +// and licensed as follows: +// +// The MIT License (MIT) +// +// Copyright (c) 2013 Hisham Muhammad +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +/* +** Adapted from Lua 5.2.0 +*/ +void luaL_setfuncs(lua_State *L, const luaL_Reg *l, int nup) { + luaL_checkstack(L, nup+1, "too many upvalues"); + for (; l->name != NULL; l++) { /* fill the table with given functions */ + int i; + lua_pushstring(L, l->name); + for (i = 0; i < nup; i++) /* copy upvalues to the top */ + lua_pushvalue(L, -(nup + 1)); + lua_pushcclosure(L, l->func, nup); /* closure with those upvalues */ + lua_settable(L, -(nup + 3)); /* table must be below the upvalues, the name and the closure */ + } + lua_pop(L, nup); /* remove upvalues */ +} + +void *luaL_testudata(lua_State *L, int i, const char *tname) { + void *p = lua_touserdata(L, i); + luaL_checkstack(L, 2, "not enough stack slots"); + if (p == NULL || !lua_getmetatable(L, i)) + return NULL; + else { + int res = 0; + luaL_getmetatable(L, tname); + res = lua_rawequal(L, -1, -2); + lua_pop(L, 2); + if (!res) + p = NULL; + } + return p; +} + +#endif + +class Mixer; + +namespace movit { +class ResourcePool; +} // namespace movit + +using namespace std; +using namespace movit; + +extern Mixer *global_mixer; + +Theme *get_theme_updata(lua_State* L) +{ + luaL_checktype(L, lua_upvalueindex(1), LUA_TLIGHTUSERDATA); + return (Theme *)lua_touserdata(L, lua_upvalueindex(1)); +} + +int ThemeMenu_set(lua_State *L) +{ + Theme *theme = get_theme_updata(L); + return theme->set_theme_menu(L); +} + +namespace { + +// Contains basically the same data as InputState, but does not hold on to +// a reference to the frames. This is important so that we can release them +// without having to wait for Lua's GC. +struct InputStateInfo { + InputStateInfo(const InputState& input_state); + + unsigned last_width[MAX_VIDEO_CARDS], last_height[MAX_VIDEO_CARDS]; + bool last_interlaced[MAX_VIDEO_CARDS], last_has_signal[MAX_VIDEO_CARDS], last_is_connected[MAX_VIDEO_CARDS]; + unsigned last_frame_rate_nom[MAX_VIDEO_CARDS], last_frame_rate_den[MAX_VIDEO_CARDS]; +}; + +InputStateInfo::InputStateInfo(const InputState &input_state) +{ + for (unsigned signal_num = 0; signal_num < MAX_VIDEO_CARDS; ++signal_num) { + BufferedFrame frame = input_state.buffered_frames[signal_num][0]; + if (frame.frame == nullptr) { + last_width[signal_num] = last_height[signal_num] = 0; + last_interlaced[signal_num] = false; + last_has_signal[signal_num] = false; + last_is_connected[signal_num] = false; + continue; + } + const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)frame.frame->userdata; + last_width[signal_num] = userdata->last_width[frame.field_number]; + last_height[signal_num] = userdata->last_height[frame.field_number]; + last_interlaced[signal_num] = userdata->last_interlaced; + last_has_signal[signal_num] = userdata->last_has_signal; + last_is_connected[signal_num] = userdata->last_is_connected; + last_frame_rate_nom[signal_num] = userdata->last_frame_rate_nom; + last_frame_rate_den[signal_num] = userdata->last_frame_rate_den; + } +} + +class LuaRefWithDeleter { +public: + LuaRefWithDeleter(mutex *m, lua_State *L, int ref) : m(m), L(L), ref(ref) {} + ~LuaRefWithDeleter() { + unique_lock lock(*m); + luaL_unref(L, LUA_REGISTRYINDEX, ref); + } + int get() const { return ref; } + +private: + LuaRefWithDeleter(const LuaRefWithDeleter &) = delete; + + mutex *m; + lua_State *L; + int ref; +}; + +template +int wrap_lua_object(lua_State* L, const char *class_name, Args&&... args) +{ + // Construct the C++ object and put it on the stack. + void *mem = lua_newuserdata(L, sizeof(T)); + new(mem) T(forward(args)...); + + // Look up the metatable named , and set it on the new object. + luaL_getmetatable(L, class_name); + lua_setmetatable(L, -2); + + return 1; +} + +// Like wrap_lua_object, but the object is not owned by Lua; ie. it's not freed +// by Lua GC. This is typically the case for Effects, which are owned by EffectChain +// and expected to be destructed by it. The object will be of type T** instead of T* +// when exposed to Lua. +// +// Note that we currently leak if you allocate an Effect in this way and never call +// add_effect. We should see if there's a way to e.g. set __gc on it at construction time +// and then release that once add_effect() takes ownership. +template +int wrap_lua_object_nonowned(lua_State* L, const char *class_name, Args&&... args) +{ + // Construct the pointer ot the C++ object and put it on the stack. + T **obj = (T **)lua_newuserdata(L, sizeof(T *)); + *obj = new T(forward(args)...); + + // Look up the metatable named , and set it on the new object. + luaL_getmetatable(L, class_name); + lua_setmetatable(L, -2); + + return 1; +} + +Effect *get_effect(lua_State *L, int idx) +{ + if (luaL_testudata(L, idx, "WhiteBalanceEffect") || + luaL_testudata(L, idx, "ResampleEffect") || + luaL_testudata(L, idx, "PaddingEffect") || + luaL_testudata(L, idx, "IntegralPaddingEffect") || + luaL_testudata(L, idx, "OverlayEffect") || + luaL_testudata(L, idx, "ResizeEffect") || + luaL_testudata(L, idx, "MultiplyEffect") || + luaL_testudata(L, idx, "MixEffect") || + luaL_testudata(L, idx, "LiftGammaGainEffect") || + luaL_testudata(L, idx, "ImageInput")) { + return *(Effect **)lua_touserdata(L, idx); + } + luaL_error(L, "Error: Index #%d was not an Effect type\n", idx); + return nullptr; +} + +InputStateInfo *get_input_state_info(lua_State *L, int idx) +{ + if (luaL_testudata(L, idx, "InputStateInfo")) { + return (InputStateInfo *)lua_touserdata(L, idx); + } + luaL_error(L, "Error: Index #%d was not InputStateInfo\n", idx); + return nullptr; +} + +bool checkbool(lua_State* L, int idx) +{ + luaL_checktype(L, idx, LUA_TBOOLEAN); + return lua_toboolean(L, idx); +} + +string checkstdstring(lua_State *L, int index) +{ + size_t len; + const char* cstr = lua_tolstring(L, index, &len); + return string(cstr, len); +} + +int EffectChain_new(lua_State* L) +{ + assert(lua_gettop(L) == 2); + Theme *theme = get_theme_updata(L); + int aspect_w = luaL_checknumber(L, 1); + int aspect_h = luaL_checknumber(L, 2); + + return wrap_lua_object(L, "EffectChain", aspect_w, aspect_h, theme->get_resource_pool()); +} + +int EffectChain_gc(lua_State* L) +{ + assert(lua_gettop(L) == 1); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + chain->~EffectChain(); + return 0; +} + +int EffectChain_add_live_input(lua_State* L) +{ + assert(lua_gettop(L) == 3); + Theme *theme = get_theme_updata(L); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + bool override_bounce = checkbool(L, 2); + bool deinterlace = checkbool(L, 3); + bmusb::PixelFormat pixel_format = global_flags.ten_bit_input ? bmusb::PixelFormat_10BitYCbCr : bmusb::PixelFormat_8BitYCbCr; + + // Needs to be nonowned to match add_video_input (see below). + return wrap_lua_object_nonowned(L, "LiveInputWrapper", theme, chain, pixel_format, override_bounce, deinterlace, /*user_connectable=*/true); +} + +int EffectChain_add_video_input(lua_State* L) +{ + assert(lua_gettop(L) == 3); + Theme *theme = get_theme_updata(L); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + FFmpegCapture **capture = (FFmpegCapture **)luaL_checkudata(L, 2, "VideoInput"); + bool deinterlace = checkbool(L, 3); + + // These need to be nonowned, so that the LiveInputWrapper still exists + // and can feed frames to the right EffectChain even if the Lua code + // doesn't care about the object anymore. (If we change this, we'd need + // to also unregister the signal connection on __gc.) + int ret = wrap_lua_object_nonowned( + L, "LiveInputWrapper", theme, chain, (*capture)->get_current_pixel_format(), + /*override_bounce=*/false, deinterlace, /*user_connectable=*/false); + if (ret == 1) { + Theme *theme = get_theme_updata(L); + LiveInputWrapper **live_input = (LiveInputWrapper **)lua_touserdata(L, -1); + theme->register_video_signal_connection(chain, *live_input, *capture); + } + return ret; +} + +#ifdef HAVE_CEF +int EffectChain_add_html_input(lua_State* L) +{ + assert(lua_gettop(L) == 2); + Theme *theme = get_theme_updata(L); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + CEFCapture **capture = (CEFCapture **)luaL_checkudata(L, 2, "HTMLInput"); + + // These need to be nonowned, so that the LiveInputWrapper still exists + // and can feed frames to the right EffectChain even if the Lua code + // doesn't care about the object anymore. (If we change this, we'd need + // to also unregister the signal connection on __gc.) + int ret = wrap_lua_object_nonowned( + L, "LiveInputWrapper", theme, chain, (*capture)->get_current_pixel_format(), + /*override_bounce=*/false, /*deinterlace=*/false, /*user_connectable=*/false); + if (ret == 1) { + Theme *theme = get_theme_updata(L); + LiveInputWrapper **live_input = (LiveInputWrapper **)lua_touserdata(L, -1); + theme->register_html_signal_connection(chain, *live_input, *capture); + } + return ret; +} +#endif + +int EffectChain_add_effect(lua_State* L) +{ + assert(lua_gettop(L) >= 2); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + + // TODO: Better error reporting. + Effect *effect = get_effect(L, 2); + if (lua_gettop(L) == 2) { + if (effect->num_inputs() == 0) { + chain->add_input((Input *)effect); + } else { + chain->add_effect(effect); + } + } else { + vector inputs; + for (int idx = 3; idx <= lua_gettop(L); ++idx) { + if (luaL_testudata(L, idx, "LiveInputWrapper")) { + LiveInputWrapper **input = (LiveInputWrapper **)lua_touserdata(L, idx); + inputs.push_back((*input)->get_effect()); + } else { + inputs.push_back(get_effect(L, idx)); + } + } + chain->add_effect(effect, inputs); + } + + lua_settop(L, 2); // Return the effect itself. + + // Make sure Lua doesn't garbage-collect it away. + lua_pushvalue(L, -1); + luaL_ref(L, LUA_REGISTRYINDEX); // TODO: leak? + + return 1; +} + +int EffectChain_finalize(lua_State* L) +{ + assert(lua_gettop(L) == 2); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + bool is_main_chain = checkbool(L, 2); + + // Add outputs as needed. + // NOTE: If you change any details about the output format, you will need to + // also update what's given to the muxer (HTTPD::Mux constructor) and + // what's put in the H.264 stream (sps_rbsp()). + ImageFormat inout_format; + inout_format.color_space = COLORSPACE_REC_709; + + // Output gamma is tricky. We should output Rec. 709 for TV, except that + // we expect to run with web players and others that don't really care and + // just output with no conversion. So that means we'll need to output sRGB, + // even though H.264 has no setting for that (we use “unspecified”). + inout_format.gamma_curve = GAMMA_sRGB; + + if (is_main_chain) { + YCbCrFormat output_ycbcr_format; + // We actually output 4:2:0 and/or 4:2:2 in the end, but chroma subsampling + // happens in a pass not run by Movit (see ChromaSubsampler::subsample_chroma()). + output_ycbcr_format.chroma_subsampling_x = 1; + output_ycbcr_format.chroma_subsampling_y = 1; + + // This will be overridden if HDMI/SDI output is in force. + if (global_flags.ycbcr_rec709_coefficients) { + output_ycbcr_format.luma_coefficients = YCBCR_REC_709; + } else { + output_ycbcr_format.luma_coefficients = YCBCR_REC_601; + } + + output_ycbcr_format.full_range = false; + output_ycbcr_format.num_levels = 1 << global_flags.x264_bit_depth; + + GLenum type = global_flags.x264_bit_depth > 8 ? GL_UNSIGNED_SHORT : GL_UNSIGNED_BYTE; + + chain->add_ycbcr_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, output_ycbcr_format, YCBCR_OUTPUT_SPLIT_Y_AND_CBCR, type); + + // If we're using zerocopy video encoding (so the destination + // Y texture is owned by VA-API and will be unavailable for + // display), add a copy, where we'll only be using the Y component. + if (global_flags.use_zerocopy) { + chain->add_ycbcr_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, output_ycbcr_format, YCBCR_OUTPUT_INTERLEAVED, type); // Add a copy where we'll only be using the Y component. + } + chain->set_dither_bits(global_flags.x264_bit_depth > 8 ? 16 : 8); + chain->set_output_origin(OUTPUT_ORIGIN_TOP_LEFT); + } else { + chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED); + } + + chain->finalize(); + return 0; +} + +int LiveInputWrapper_connect_signal(lua_State* L) +{ + assert(lua_gettop(L) == 2); + LiveInputWrapper **input = (LiveInputWrapper **)luaL_checkudata(L, 1, "LiveInputWrapper"); + int signal_num = luaL_checknumber(L, 2); + bool success = (*input)->connect_signal(signal_num); + if (!success) { + lua_Debug ar; + lua_getstack(L, 1, &ar); + lua_getinfo(L, "nSl", &ar); + fprintf(stderr, "ERROR: %s:%d: Calling connect_signal() on a video or HTML input. Ignoring.\n", + ar.source, ar.currentline); + } + return 0; +} + +int ImageInput_new(lua_State* L) +{ + assert(lua_gettop(L) == 1); + string filename = checkstdstring(L, 1); + return wrap_lua_object_nonowned(L, "ImageInput", filename); +} + +int VideoInput_new(lua_State* L) +{ + assert(lua_gettop(L) == 2); + string filename = checkstdstring(L, 1); + int pixel_format = luaL_checknumber(L, 2); + if (pixel_format != bmusb::PixelFormat_8BitYCbCrPlanar && + pixel_format != bmusb::PixelFormat_8BitBGRA) { + fprintf(stderr, "WARNING: Invalid enum %d used for video format, choosing Y'CbCr.\n", + pixel_format); + pixel_format = bmusb::PixelFormat_8BitYCbCrPlanar; + } + int ret = wrap_lua_object_nonowned(L, "VideoInput", filename, global_flags.width, global_flags.height); + if (ret == 1) { + FFmpegCapture **capture = (FFmpegCapture **)lua_touserdata(L, -1); + (*capture)->set_pixel_format(bmusb::PixelFormat(pixel_format)); + + Theme *theme = get_theme_updata(L); + theme->register_video_input(*capture); + } + return ret; +} + +int VideoInput_rewind(lua_State* L) +{ + assert(lua_gettop(L) == 1); + FFmpegCapture **video_input = (FFmpegCapture **)luaL_checkudata(L, 1, "VideoInput"); + (*video_input)->rewind(); + return 0; +} + +int VideoInput_disconnect(lua_State* L) +{ + assert(lua_gettop(L) == 1); + FFmpegCapture **video_input = (FFmpegCapture **)luaL_checkudata(L, 1, "VideoInput"); + (*video_input)->disconnect(); + return 0; +} + +int VideoInput_change_rate(lua_State* L) +{ + assert(lua_gettop(L) == 2); + FFmpegCapture **video_input = (FFmpegCapture **)luaL_checkudata(L, 1, "VideoInput"); + double new_rate = luaL_checknumber(L, 2); + (*video_input)->change_rate(new_rate); + return 0; +} + +int VideoInput_get_signal_num(lua_State* L) +{ + assert(lua_gettop(L) == 1); + FFmpegCapture **video_input = (FFmpegCapture **)luaL_checkudata(L, 1, "VideoInput"); + lua_pushnumber(L, -1 - (*video_input)->get_card_index()); + return 1; +} + +int HTMLInput_new(lua_State* L) +{ +#ifdef HAVE_CEF + assert(lua_gettop(L) == 1); + string url = checkstdstring(L, 1); + int ret = wrap_lua_object_nonowned(L, "HTMLInput", url, global_flags.width, global_flags.height); + if (ret == 1) { + CEFCapture **capture = (CEFCapture **)lua_touserdata(L, -1); + Theme *theme = get_theme_updata(L); + theme->register_html_input(*capture); + } + return ret; +#else + fprintf(stderr, "This version of Nageru has been compiled without CEF support.\n"); + fprintf(stderr, "HTMLInput is not available.\n"); + exit(1); +#endif +} + +#ifdef HAVE_CEF +int HTMLInput_set_url(lua_State* L) +{ + assert(lua_gettop(L) == 2); + CEFCapture **video_input = (CEFCapture **)luaL_checkudata(L, 1, "HTMLInput"); + string new_url = checkstdstring(L, 2); + (*video_input)->set_url(new_url); + return 0; +} + +int HTMLInput_reload(lua_State* L) +{ + assert(lua_gettop(L) == 1); + CEFCapture **video_input = (CEFCapture **)luaL_checkudata(L, 1, "HTMLInput"); + (*video_input)->reload(); + return 0; +} + +int HTMLInput_set_max_fps(lua_State* L) +{ + assert(lua_gettop(L) == 2); + CEFCapture **video_input = (CEFCapture **)luaL_checkudata(L, 1, "HTMLInput"); + int max_fps = lrint(luaL_checknumber(L, 2)); + (*video_input)->set_max_fps(max_fps); + return 0; +} + +int HTMLInput_execute_javascript_async(lua_State* L) +{ + assert(lua_gettop(L) == 2); + CEFCapture **video_input = (CEFCapture **)luaL_checkudata(L, 1, "HTMLInput"); + string js = checkstdstring(L, 2); + (*video_input)->execute_javascript_async(js); + return 0; +} + +int HTMLInput_resize(lua_State* L) +{ + assert(lua_gettop(L) == 3); + CEFCapture **video_input = (CEFCapture **)luaL_checkudata(L, 1, "HTMLInput"); + unsigned width = lrint(luaL_checknumber(L, 2)); + unsigned height = lrint(luaL_checknumber(L, 3)); + (*video_input)->resize(width, height); + return 0; +} + +int HTMLInput_get_signal_num(lua_State* L) +{ + assert(lua_gettop(L) == 1); + CEFCapture **video_input = (CEFCapture **)luaL_checkudata(L, 1, "HTMLInput"); + lua_pushnumber(L, -1 - (*video_input)->get_card_index()); + return 1; +} +#endif + +int WhiteBalanceEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "WhiteBalanceEffect"); +} + +int ResampleEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "ResampleEffect"); +} + +int PaddingEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "PaddingEffect"); +} + +int IntegralPaddingEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "IntegralPaddingEffect"); +} + +int OverlayEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "OverlayEffect"); +} + +int ResizeEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "ResizeEffect"); +} + +int MultiplyEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "MultiplyEffect"); +} + +int MixEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "MixEffect"); +} + +int LiftGammaGainEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object_nonowned(L, "LiftGammaGainEffect"); +} + +int InputStateInfo_get_width(lua_State* L) +{ + assert(lua_gettop(L) == 2); + InputStateInfo *input_state_info = get_input_state_info(L, 1); + Theme *theme = get_theme_updata(L); + int signal_num = theme->map_signal(luaL_checknumber(L, 2)); + lua_pushnumber(L, input_state_info->last_width[signal_num]); + return 1; +} + +int InputStateInfo_get_height(lua_State* L) +{ + assert(lua_gettop(L) == 2); + InputStateInfo *input_state_info = get_input_state_info(L, 1); + Theme *theme = get_theme_updata(L); + int signal_num = theme->map_signal(luaL_checknumber(L, 2)); + lua_pushnumber(L, input_state_info->last_height[signal_num]); + return 1; +} + +int InputStateInfo_get_interlaced(lua_State* L) +{ + assert(lua_gettop(L) == 2); + InputStateInfo *input_state_info = get_input_state_info(L, 1); + Theme *theme = get_theme_updata(L); + int signal_num = theme->map_signal(luaL_checknumber(L, 2)); + lua_pushboolean(L, input_state_info->last_interlaced[signal_num]); + return 1; +} + +int InputStateInfo_get_has_signal(lua_State* L) +{ + assert(lua_gettop(L) == 2); + InputStateInfo *input_state_info = get_input_state_info(L, 1); + Theme *theme = get_theme_updata(L); + int signal_num = theme->map_signal(luaL_checknumber(L, 2)); + lua_pushboolean(L, input_state_info->last_has_signal[signal_num]); + return 1; +} + +int InputStateInfo_get_is_connected(lua_State* L) +{ + assert(lua_gettop(L) == 2); + InputStateInfo *input_state_info = get_input_state_info(L, 1); + Theme *theme = get_theme_updata(L); + int signal_num = theme->map_signal(luaL_checknumber(L, 2)); + lua_pushboolean(L, input_state_info->last_is_connected[signal_num]); + return 1; +} + +int InputStateInfo_get_frame_rate_nom(lua_State* L) +{ + assert(lua_gettop(L) == 2); + InputStateInfo *input_state_info = get_input_state_info(L, 1); + Theme *theme = get_theme_updata(L); + int signal_num = theme->map_signal(luaL_checknumber(L, 2)); + lua_pushnumber(L, input_state_info->last_frame_rate_nom[signal_num]); + return 1; +} + +int InputStateInfo_get_frame_rate_den(lua_State* L) +{ + assert(lua_gettop(L) == 2); + InputStateInfo *input_state_info = get_input_state_info(L, 1); + Theme *theme = get_theme_updata(L); + int signal_num = theme->map_signal(luaL_checknumber(L, 2)); + lua_pushnumber(L, input_state_info->last_frame_rate_den[signal_num]); + return 1; +} + +int Effect_set_float(lua_State *L) +{ + assert(lua_gettop(L) == 3); + Effect *effect = (Effect *)get_effect(L, 1); + string key = checkstdstring(L, 2); + float value = luaL_checknumber(L, 3); + if (!effect->set_float(key, value)) { + luaL_error(L, "Effect refused set_float(\"%s\", %d) (invalid key?)", key.c_str(), int(value)); + } + return 0; +} + +int Effect_set_int(lua_State *L) +{ + assert(lua_gettop(L) == 3); + Effect *effect = (Effect *)get_effect(L, 1); + string key = checkstdstring(L, 2); + float value = luaL_checknumber(L, 3); + if (!effect->set_int(key, value)) { + luaL_error(L, "Effect refused set_int(\"%s\", %d) (invalid key?)", key.c_str(), int(value)); + } + return 0; +} + +int Effect_set_vec3(lua_State *L) +{ + assert(lua_gettop(L) == 5); + Effect *effect = (Effect *)get_effect(L, 1); + string key = checkstdstring(L, 2); + float v[3]; + v[0] = luaL_checknumber(L, 3); + v[1] = luaL_checknumber(L, 4); + v[2] = luaL_checknumber(L, 5); + if (!effect->set_vec3(key, v)) { + luaL_error(L, "Effect refused set_vec3(\"%s\", %f, %f, %f) (invalid key?)", key.c_str(), + v[0], v[1], v[2]); + } + return 0; +} + +int Effect_set_vec4(lua_State *L) +{ + assert(lua_gettop(L) == 6); + Effect *effect = (Effect *)get_effect(L, 1); + string key = checkstdstring(L, 2); + float v[4]; + v[0] = luaL_checknumber(L, 3); + v[1] = luaL_checknumber(L, 4); + v[2] = luaL_checknumber(L, 5); + v[3] = luaL_checknumber(L, 6); + if (!effect->set_vec4(key, v)) { + luaL_error(L, "Effect refused set_vec4(\"%s\", %f, %f, %f, %f) (invalid key?)", key.c_str(), + v[0], v[1], v[2], v[3]); + } + return 0; +} + +const luaL_Reg EffectChain_funcs[] = { + { "new", EffectChain_new }, + { "__gc", EffectChain_gc }, + { "add_live_input", EffectChain_add_live_input }, + { "add_video_input", EffectChain_add_video_input }, +#ifdef HAVE_CEF + { "add_html_input", EffectChain_add_html_input }, +#endif + { "add_effect", EffectChain_add_effect }, + { "finalize", EffectChain_finalize }, + { NULL, NULL } +}; + +const luaL_Reg LiveInputWrapper_funcs[] = { + { "connect_signal", LiveInputWrapper_connect_signal }, + { NULL, NULL } +}; + +const luaL_Reg ImageInput_funcs[] = { + { "new", ImageInput_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg VideoInput_funcs[] = { + { "new", VideoInput_new }, + { "rewind", VideoInput_rewind }, + { "disconnect", VideoInput_disconnect }, + { "change_rate", VideoInput_change_rate }, + { "get_signal_num", VideoInput_get_signal_num }, + { NULL, NULL } +}; + +const luaL_Reg HTMLInput_funcs[] = { + { "new", HTMLInput_new }, +#ifdef HAVE_CEF + { "set_url", HTMLInput_set_url }, + { "reload", HTMLInput_reload }, + { "set_max_fps", HTMLInput_set_max_fps }, + { "execute_javascript_async", HTMLInput_execute_javascript_async }, + { "resize", HTMLInput_resize }, + { "get_signal_num", HTMLInput_get_signal_num }, +#endif + { NULL, NULL } +}; + +const luaL_Reg WhiteBalanceEffect_funcs[] = { + { "new", WhiteBalanceEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg ResampleEffect_funcs[] = { + { "new", ResampleEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg PaddingEffect_funcs[] = { + { "new", PaddingEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg IntegralPaddingEffect_funcs[] = { + { "new", IntegralPaddingEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg OverlayEffect_funcs[] = { + { "new", OverlayEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg ResizeEffect_funcs[] = { + { "new", ResizeEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg MultiplyEffect_funcs[] = { + { "new", MultiplyEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg MixEffect_funcs[] = { + { "new", MixEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg LiftGammaGainEffect_funcs[] = { + { "new", LiftGammaGainEffect_new }, + { "set_float", Effect_set_float }, + { "set_int", Effect_set_int }, + { "set_vec3", Effect_set_vec3 }, + { "set_vec4", Effect_set_vec4 }, + { NULL, NULL } +}; + +const luaL_Reg InputStateInfo_funcs[] = { + { "get_width", InputStateInfo_get_width }, + { "get_height", InputStateInfo_get_height }, + { "get_interlaced", InputStateInfo_get_interlaced }, + { "get_has_signal", InputStateInfo_get_has_signal }, + { "get_is_connected", InputStateInfo_get_is_connected }, + { "get_frame_rate_nom", InputStateInfo_get_frame_rate_nom }, + { "get_frame_rate_den", InputStateInfo_get_frame_rate_den }, + { NULL, NULL } +}; + +const luaL_Reg ThemeMenu_funcs[] = { + { "set", ThemeMenu_set }, + { NULL, NULL } +}; + +} // namespace + +LiveInputWrapper::LiveInputWrapper( + Theme *theme, + EffectChain *chain, + bmusb::PixelFormat pixel_format, + bool override_bounce, + bool deinterlace, + bool user_connectable) + : theme(theme), + pixel_format(pixel_format), + deinterlace(deinterlace), + user_connectable(user_connectable) +{ + ImageFormat inout_format; + inout_format.color_space = COLORSPACE_sRGB; + + // Gamma curve depends on the input signal, and we don't really get any + // indications. A camera would be expected to do Rec. 709, but + // I haven't checked if any do in practice. However, computers _do_ output + // in sRGB gamma (ie., they don't convert from sRGB to Rec. 709), and + // I wouldn't really be surprised if most non-professional cameras do, too. + // So we pick sRGB as the least evil here. + inout_format.gamma_curve = GAMMA_sRGB; + + unsigned num_inputs; + if (deinterlace) { + deinterlace_effect = new movit::DeinterlaceEffect(); + + // As per the comments in deinterlace_effect.h, we turn this off. + // The most likely interlaced input for us is either a camera + // (where it's fine to turn it off) or a laptop (where it _should_ + // be turned off). + CHECK(deinterlace_effect->set_int("enable_spatial_interlacing_check", 0)); + + num_inputs = deinterlace_effect->num_inputs(); + assert(num_inputs == FRAME_HISTORY_LENGTH); + } else { + num_inputs = 1; + } + + if (pixel_format == bmusb::PixelFormat_8BitBGRA) { + for (unsigned i = 0; i < num_inputs; ++i) { + // We upload our textures ourselves, and Movit swaps + // R and B in the shader if we specify BGRA, so lie and say RGBA. + if (global_flags.can_disable_srgb_decoder) { + rgba_inputs.push_back(new sRGBSwitchingFlatInput(inout_format, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, GL_UNSIGNED_BYTE, global_flags.width, global_flags.height)); + } else { + rgba_inputs.push_back(new NonsRGBCapableFlatInput(inout_format, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, GL_UNSIGNED_BYTE, global_flags.width, global_flags.height)); + } + chain->add_input(rgba_inputs.back()); + } + + if (deinterlace) { + vector reverse_inputs(rgba_inputs.rbegin(), rgba_inputs.rend()); + chain->add_effect(deinterlace_effect, reverse_inputs); + } + } else { + assert(pixel_format == bmusb::PixelFormat_8BitYCbCr || + pixel_format == bmusb::PixelFormat_10BitYCbCr || + pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); + + // Most of these settings will be overridden later if using PixelFormat_8BitYCbCrPlanar. + input_ycbcr_format.chroma_subsampling_x = (pixel_format == bmusb::PixelFormat_10BitYCbCr) ? 1 : 2; + input_ycbcr_format.chroma_subsampling_y = 1; + input_ycbcr_format.num_levels = (pixel_format == bmusb::PixelFormat_10BitYCbCr) ? 1024 : 256; + input_ycbcr_format.cb_x_position = 0.0; + input_ycbcr_format.cr_x_position = 0.0; + input_ycbcr_format.cb_y_position = 0.5; + input_ycbcr_format.cr_y_position = 0.5; + input_ycbcr_format.luma_coefficients = YCBCR_REC_709; // Will be overridden later even if not planar. + input_ycbcr_format.full_range = false; // Will be overridden later even if not planar. + + for (unsigned i = 0; i < num_inputs; ++i) { + // When using 10-bit input, we're converting to interleaved through v210Converter. + YCbCrInputSplitting splitting; + if (pixel_format == bmusb::PixelFormat_10BitYCbCr) { + splitting = YCBCR_INPUT_INTERLEAVED; + } else if (pixel_format == bmusb::PixelFormat_8BitYCbCr) { + splitting = YCBCR_INPUT_SPLIT_Y_AND_CBCR; + } else { + splitting = YCBCR_INPUT_PLANAR; + } + if (override_bounce) { + ycbcr_inputs.push_back(new NonBouncingYCbCrInput(inout_format, input_ycbcr_format, global_flags.width, global_flags.height, splitting)); + } else { + ycbcr_inputs.push_back(new YCbCrInput(inout_format, input_ycbcr_format, global_flags.width, global_flags.height, splitting)); + } + chain->add_input(ycbcr_inputs.back()); + } + + if (deinterlace) { + vector reverse_inputs(ycbcr_inputs.rbegin(), ycbcr_inputs.rend()); + chain->add_effect(deinterlace_effect, reverse_inputs); + } + } +} + +bool LiveInputWrapper::connect_signal(int signal_num) +{ + if (!user_connectable) { + return false; + } + + if (global_mixer == nullptr) { + // No data yet. + return true; + } + + signal_num = theme->map_signal(signal_num); + connect_signal_raw(signal_num, *theme->input_state); + return true; +} + +void LiveInputWrapper::connect_signal_raw(int signal_num, const InputState &input_state) +{ + BufferedFrame first_frame = input_state.buffered_frames[signal_num][0]; + if (first_frame.frame == nullptr) { + // No data yet. + return; + } + unsigned width, height; + { + const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)first_frame.frame->userdata; + width = userdata->last_width[first_frame.field_number]; + height = userdata->last_height[first_frame.field_number]; + if (userdata->last_interlaced) { + height *= 2; + } + } + + movit::YCbCrLumaCoefficients ycbcr_coefficients = input_state.ycbcr_coefficients[signal_num]; + bool full_range = input_state.full_range[signal_num]; + + if (input_state.ycbcr_coefficients_auto[signal_num]) { + full_range = false; + + // The Blackmagic driver docs claim that the device outputs Y'CbCr + // according to Rec. 601, but this seems to indicate the subsampling + // positions only, as they publish Y'CbCr → RGB formulas that are + // different for HD and SD (corresponding to Rec. 709 and 601, respectively), + // and a Lenovo X1 gen 3 I used to test definitely outputs Rec. 709 + // (at least up to rounding error). Other devices seem to use Rec. 601 + // even on HD resolutions. Nevertheless, Rec. 709 _is_ the right choice + // for HD, so we default to that if the user hasn't set anything. + if (height >= 720) { + ycbcr_coefficients = YCBCR_REC_709; + } else { + ycbcr_coefficients = YCBCR_REC_601; + } + } + + // This is a global, but it doesn't really matter. + input_ycbcr_format.luma_coefficients = ycbcr_coefficients; + input_ycbcr_format.full_range = full_range; + + BufferedFrame last_good_frame = first_frame; + for (unsigned i = 0; i < max(ycbcr_inputs.size(), rgba_inputs.size()); ++i) { + BufferedFrame frame = input_state.buffered_frames[signal_num][i]; + if (frame.frame == nullptr) { + // Not enough data; reuse last frame (well, field). + // This is suboptimal, but we have nothing better. + frame = last_good_frame; + } + const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)frame.frame->userdata; + + unsigned this_width = userdata->last_width[frame.field_number]; + unsigned this_height = userdata->last_height[frame.field_number]; + if (this_width != width || this_height != height) { + // Resolution changed; reuse last frame/field. + frame = last_good_frame; + userdata = (const PBOFrameAllocator::Userdata *)frame.frame->userdata; + } + + assert(userdata->pixel_format == pixel_format); + switch (pixel_format) { + case bmusb::PixelFormat_8BitYCbCr: + ycbcr_inputs[i]->set_texture_num(0, userdata->tex_y[frame.field_number]); + ycbcr_inputs[i]->set_texture_num(1, userdata->tex_cbcr[frame.field_number]); + ycbcr_inputs[i]->change_ycbcr_format(input_ycbcr_format); + ycbcr_inputs[i]->set_width(width); + ycbcr_inputs[i]->set_height(height); + break; + case bmusb::PixelFormat_8BitYCbCrPlanar: + ycbcr_inputs[i]->set_texture_num(0, userdata->tex_y[frame.field_number]); + ycbcr_inputs[i]->set_texture_num(1, userdata->tex_cb[frame.field_number]); + ycbcr_inputs[i]->set_texture_num(2, userdata->tex_cr[frame.field_number]); + ycbcr_inputs[i]->change_ycbcr_format(userdata->ycbcr_format); + ycbcr_inputs[i]->set_width(width); + ycbcr_inputs[i]->set_height(height); + break; + case bmusb::PixelFormat_10BitYCbCr: + ycbcr_inputs[i]->set_texture_num(0, userdata->tex_444[frame.field_number]); + ycbcr_inputs[i]->change_ycbcr_format(input_ycbcr_format); + ycbcr_inputs[i]->set_width(width); + ycbcr_inputs[i]->set_height(height); + break; + case bmusb::PixelFormat_8BitBGRA: + rgba_inputs[i]->set_texture_num(userdata->tex_rgba[frame.field_number]); + rgba_inputs[i]->set_width(width); + rgba_inputs[i]->set_height(height); + break; + default: + assert(false); + } + + last_good_frame = frame; + } + + if (deinterlace) { + BufferedFrame frame = input_state.buffered_frames[signal_num][0]; + CHECK(deinterlace_effect->set_int("current_field_position", frame.field_number)); + } +} + +namespace { + +int call_num_channels(lua_State *L) +{ + lua_getglobal(L, "num_channels"); + + if (lua_pcall(L, 0, 1, 0) != 0) { + fprintf(stderr, "error running function `num_channels': %s\n", lua_tostring(L, -1)); + exit(1); + } + + int num_channels = luaL_checknumber(L, 1); + lua_pop(L, 1); + assert(lua_gettop(L) == 0); + return num_channels; +} + +} // namespace + +Theme::Theme(const string &filename, const vector &search_dirs, ResourcePool *resource_pool, unsigned num_cards) + : resource_pool(resource_pool), num_cards(num_cards), signal_to_card_mapping(global_flags.default_stream_mapping) +{ + L = luaL_newstate(); + luaL_openlibs(L); + + // Search through all directories until we find a file that will load + // (as in, does not return LUA_ERRFILE); then run it. We store load errors + // from all the attempts, and show them once we know we can't find any of them. + lua_settop(L, 0); + vector errors; + bool success = false; + + vector real_search_dirs; + if (!filename.empty() && filename[0] == '/') { + real_search_dirs.push_back(""); + } else { + real_search_dirs = search_dirs; + } + + string path; + int theme_code_ref; + for (const string &dir : real_search_dirs) { + if (dir.empty()) { + path = filename; + } else { + path = dir + "/" + filename; + } + int err = luaL_loadfile(L, path.c_str()); + if (err == 0) { + // Save the theme for when we're actually going to run it + // (we need to set up the right environment below first, + // and we couldn't do that before, because we didn't know the + // path to put in Nageru.THEME_PATH). + theme_code_ref = luaL_ref(L, LUA_REGISTRYINDEX); + assert(lua_gettop(L) == 0); + + success = true; + break; + } + errors.push_back(lua_tostring(L, -1)); + lua_pop(L, 1); + if (err != LUA_ERRFILE) { + // The file actually loaded, but failed to parse somehow. Abort; don't try the next one. + break; + } + } + + if (!success) { + for (const string &error : errors) { + fprintf(stderr, "%s\n", error.c_str()); + } + exit(1); + } + assert(lua_gettop(L) == 0); + + // Make sure the path exposed to the theme (as Nageru.THEME_PATH; + // can be useful for locating files when talking to CEF) is absolute. + // In a sense, it would be nice if realpath() had a mode not to + // resolve symlinks, but it doesn't, so we only call it if we don't + // already have an absolute path (which may leave ../ elements etc.). + if (path[0] == '/') { + theme_path = path; + } else { + char *absolute_theme_path = realpath(path.c_str(), nullptr); + theme_path = absolute_theme_path; + free(absolute_theme_path); + } + + // Set up the API we provide. + register_constants(); + register_class("EffectChain", EffectChain_funcs); + register_class("LiveInputWrapper", LiveInputWrapper_funcs); + register_class("ImageInput", ImageInput_funcs); + register_class("VideoInput", VideoInput_funcs); + register_class("HTMLInput", HTMLInput_funcs); + register_class("WhiteBalanceEffect", WhiteBalanceEffect_funcs); + register_class("ResampleEffect", ResampleEffect_funcs); + register_class("PaddingEffect", PaddingEffect_funcs); + register_class("IntegralPaddingEffect", IntegralPaddingEffect_funcs); + register_class("OverlayEffect", OverlayEffect_funcs); + register_class("ResizeEffect", ResizeEffect_funcs); + register_class("MultiplyEffect", MultiplyEffect_funcs); + register_class("MixEffect", MixEffect_funcs); + register_class("LiftGammaGainEffect", LiftGammaGainEffect_funcs); + register_class("InputStateInfo", InputStateInfo_funcs); + register_class("ThemeMenu", ThemeMenu_funcs); + + // Now actually run the theme to get everything set up. + lua_rawgeti(L, LUA_REGISTRYINDEX, theme_code_ref); + luaL_unref(L, LUA_REGISTRYINDEX, theme_code_ref); + if (lua_pcall(L, 0, 0, 0)) { + fprintf(stderr, "Error when running %s: %s\n", path.c_str(), lua_tostring(L, -1)); + exit(1); + } + assert(lua_gettop(L) == 0); + + // Ask it for the number of channels. + num_channels = call_num_channels(L); +} + +Theme::~Theme() +{ + lua_close(L); +} + +void Theme::register_constants() +{ + // Set Nageru.VIDEO_FORMAT_BGRA = bmusb::PixelFormat_8BitBGRA, etc. + const vector> num_constants = { + { "VIDEO_FORMAT_BGRA", bmusb::PixelFormat_8BitBGRA }, + { "VIDEO_FORMAT_YCBCR", bmusb::PixelFormat_8BitYCbCrPlanar }, + }; + const vector> str_constants = { + { "THEME_PATH", theme_path }, + }; + + lua_newtable(L); // t = {} + + for (const pair &constant : num_constants) { + lua_pushstring(L, constant.first.c_str()); + lua_pushinteger(L, constant.second); + lua_settable(L, 1); // t[key] = value + } + for (const pair &constant : str_constants) { + lua_pushstring(L, constant.first.c_str()); + lua_pushstring(L, constant.second.c_str()); + lua_settable(L, 1); // t[key] = value + } + + lua_setglobal(L, "Nageru"); // Nageru = t + assert(lua_gettop(L) == 0); +} + +void Theme::register_class(const char *class_name, const luaL_Reg *funcs) +{ + assert(lua_gettop(L) == 0); + luaL_newmetatable(L, class_name); // mt = {} + lua_pushlightuserdata(L, this); + luaL_setfuncs(L, funcs, 1); // for (name,f in funcs) { mt[name] = f, with upvalue {theme} } + lua_pushvalue(L, -1); + lua_setfield(L, -2, "__index"); // mt.__index = mt + lua_setglobal(L, class_name); // ClassName = mt + assert(lua_gettop(L) == 0); +} + +Theme::Chain Theme::get_chain(unsigned num, float t, unsigned width, unsigned height, InputState input_state) +{ + Chain chain; + + unique_lock lock(m); + assert(lua_gettop(L) == 0); + lua_getglobal(L, "get_chain"); /* function to be called */ + lua_pushnumber(L, num); + lua_pushnumber(L, t); + lua_pushnumber(L, width); + lua_pushnumber(L, height); + wrap_lua_object(L, "InputStateInfo", input_state); + + if (lua_pcall(L, 5, 2, 0) != 0) { + fprintf(stderr, "error running function `get_chain': %s\n", lua_tostring(L, -1)); + exit(1); + } + + EffectChain *effect_chain = (EffectChain *)luaL_testudata(L, -2, "EffectChain"); + if (effect_chain == nullptr) { + fprintf(stderr, "get_chain() for chain number %d did not return an EffectChain\n", + num); + exit(1); + } + chain.chain = effect_chain; + if (!lua_isfunction(L, -1)) { + fprintf(stderr, "Argument #-1 should be a function\n"); + exit(1); + } + lua_pushvalue(L, -1); + shared_ptr funcref(new LuaRefWithDeleter(&m, L, luaL_ref(L, LUA_REGISTRYINDEX))); + lua_pop(L, 2); + assert(lua_gettop(L) == 0); + + chain.setup_chain = [this, funcref, input_state, effect_chain]{ + unique_lock lock(m); + + assert(this->input_state == nullptr); + this->input_state = &input_state; + + // Set up state, including connecting signals. + lua_rawgeti(L, LUA_REGISTRYINDEX, funcref->get()); + if (lua_pcall(L, 0, 0, 0) != 0) { + fprintf(stderr, "error running chain setup callback: %s\n", lua_tostring(L, -1)); + exit(1); + } + assert(lua_gettop(L) == 0); + + // The theme can't (or at least shouldn't!) call connect_signal() on + // each FFmpeg or CEF input, so we'll do it here. + if (video_signal_connections.count(effect_chain)) { + for (const VideoSignalConnection &conn : video_signal_connections[effect_chain]) { + conn.wrapper->connect_signal_raw(conn.source->get_card_index(), input_state); + } + } +#ifdef HAVE_CEF + if (html_signal_connections.count(effect_chain)) { + for (const CEFSignalConnection &conn : html_signal_connections[effect_chain]) { + conn.wrapper->connect_signal_raw(conn.source->get_card_index(), input_state); + } + } +#endif + + this->input_state = nullptr; + }; + + // TODO: Can we do better, e.g. by running setup_chain() and seeing what it references? + // Actually, setup_chain does maybe hold all the references we need now anyway? + chain.input_frames.reserve(num_cards * FRAME_HISTORY_LENGTH); + for (unsigned card_index = 0; card_index < num_cards; ++card_index) { + for (unsigned frame_num = 0; frame_num < FRAME_HISTORY_LENGTH; ++frame_num) { + chain.input_frames.push_back(input_state.buffered_frames[card_index][frame_num].frame); + } + } + + return chain; +} + +string Theme::get_channel_name(unsigned channel) +{ + unique_lock lock(m); + lua_getglobal(L, "channel_name"); + lua_pushnumber(L, channel); + if (lua_pcall(L, 1, 1, 0) != 0) { + fprintf(stderr, "error running function `channel_name': %s\n", lua_tostring(L, -1)); + exit(1); + } + const char *ret = lua_tostring(L, -1); + if (ret == nullptr) { + fprintf(stderr, "function `channel_name' returned nil for channel %d\n", channel); + exit(1); + } + + string retstr = ret; + lua_pop(L, 1); + assert(lua_gettop(L) == 0); + return retstr; +} + +int Theme::get_channel_signal(unsigned channel) +{ + unique_lock lock(m); + lua_getglobal(L, "channel_signal"); + lua_pushnumber(L, channel); + if (lua_pcall(L, 1, 1, 0) != 0) { + fprintf(stderr, "error running function `channel_signal': %s\n", lua_tostring(L, -1)); + exit(1); + } + + int ret = luaL_checknumber(L, 1); + lua_pop(L, 1); + assert(lua_gettop(L) == 0); + return ret; +} + +std::string Theme::get_channel_color(unsigned channel) +{ + unique_lock lock(m); + lua_getglobal(L, "channel_color"); + lua_pushnumber(L, channel); + if (lua_pcall(L, 1, 1, 0) != 0) { + fprintf(stderr, "error running function `channel_color': %s\n", lua_tostring(L, -1)); + exit(1); + } + + const char *ret = lua_tostring(L, -1); + if (ret == nullptr) { + fprintf(stderr, "function `channel_color' returned nil for channel %d\n", channel); + exit(1); + } + + string retstr = ret; + lua_pop(L, 1); + assert(lua_gettop(L) == 0); + return retstr; +} + +bool Theme::get_supports_set_wb(unsigned channel) +{ + unique_lock lock(m); + lua_getglobal(L, "supports_set_wb"); + lua_pushnumber(L, channel); + if (lua_pcall(L, 1, 1, 0) != 0) { + fprintf(stderr, "error running function `supports_set_wb': %s\n", lua_tostring(L, -1)); + exit(1); + } + + bool ret = checkbool(L, -1); + lua_pop(L, 1); + assert(lua_gettop(L) == 0); + return ret; +} + +void Theme::set_wb(unsigned channel, double r, double g, double b) +{ + unique_lock lock(m); + lua_getglobal(L, "set_wb"); + lua_pushnumber(L, channel); + lua_pushnumber(L, r); + lua_pushnumber(L, g); + lua_pushnumber(L, b); + if (lua_pcall(L, 4, 0, 0) != 0) { + fprintf(stderr, "error running function `set_wb': %s\n", lua_tostring(L, -1)); + exit(1); + } + + assert(lua_gettop(L) == 0); +} + +vector Theme::get_transition_names(float t) +{ + unique_lock lock(m); + lua_getglobal(L, "get_transitions"); + lua_pushnumber(L, t); + if (lua_pcall(L, 1, 1, 0) != 0) { + fprintf(stderr, "error running function `get_transitions': %s\n", lua_tostring(L, -1)); + exit(1); + } + + vector ret; + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + ret.push_back(lua_tostring(L, -1)); + lua_pop(L, 1); + } + lua_pop(L, 1); + assert(lua_gettop(L) == 0); + return ret; +} + +int Theme::map_signal(int signal_num) +{ + // Negative numbers map to raw signals. + if (signal_num < 0) { + return -1 - signal_num; + } + + unique_lock lock(map_m); + if (signal_to_card_mapping.count(signal_num)) { + return signal_to_card_mapping[signal_num]; + } + + int card_index; + if (global_flags.output_card != -1 && num_cards > 1) { + // Try to exclude the output card from the default card_index. + card_index = signal_num % (num_cards - 1); + if (card_index >= global_flags.output_card) { + ++card_index; + } + if (signal_num >= int(num_cards - 1)) { + fprintf(stderr, "WARNING: Theme asked for input %d, but we only have %u input card(s) (card %d is busy with output).\n", + signal_num, num_cards - 1, global_flags.output_card); + fprintf(stderr, "Mapping to card %d instead.\n", card_index); + } + } else { + card_index = signal_num % num_cards; + if (signal_num >= int(num_cards)) { + fprintf(stderr, "WARNING: Theme asked for input %d, but we only have %u card(s).\n", signal_num, num_cards); + fprintf(stderr, "Mapping to card %d instead.\n", card_index); + } + } + signal_to_card_mapping[signal_num] = card_index; + return card_index; +} + +void Theme::set_signal_mapping(int signal_num, int card_num) +{ + unique_lock lock(map_m); + assert(card_num < int(num_cards)); + signal_to_card_mapping[signal_num] = card_num; +} + +void Theme::transition_clicked(int transition_num, float t) +{ + unique_lock lock(m); + lua_getglobal(L, "transition_clicked"); + lua_pushnumber(L, transition_num); + lua_pushnumber(L, t); + + if (lua_pcall(L, 2, 0, 0) != 0) { + fprintf(stderr, "error running function `transition_clicked': %s\n", lua_tostring(L, -1)); + exit(1); + } + assert(lua_gettop(L) == 0); +} + +void Theme::channel_clicked(int preview_num) +{ + unique_lock lock(m); + lua_getglobal(L, "channel_clicked"); + lua_pushnumber(L, preview_num); + + if (lua_pcall(L, 1, 0, 0) != 0) { + fprintf(stderr, "error running function `channel_clicked': %s\n", lua_tostring(L, -1)); + exit(1); + } + assert(lua_gettop(L) == 0); +} + +int Theme::set_theme_menu(lua_State *L) +{ + for (const Theme::MenuEntry &entry : theme_menu) { + luaL_unref(L, LUA_REGISTRYINDEX, entry.lua_ref); + } + theme_menu.clear(); + + int num_elements = lua_gettop(L); + for (int i = 1; i <= num_elements; ++i) { + lua_rawgeti(L, i, 1); + const string text = checkstdstring(L, -1); + lua_pop(L, 1); + + lua_rawgeti(L, i, 2); + luaL_checktype(L, -1, LUA_TFUNCTION); + int ref = luaL_ref(L, LUA_REGISTRYINDEX); + + theme_menu.push_back(MenuEntry{ text, ref }); + } + lua_pop(L, num_elements); + assert(lua_gettop(L) == 0); + + if (theme_menu_callback != nullptr) { + theme_menu_callback(); + } + + return 0; +} + +void Theme::theme_menu_entry_clicked(int lua_ref) +{ + unique_lock lock(m); + lua_rawgeti(L, LUA_REGISTRYINDEX, lua_ref); + if (lua_pcall(L, 0, 0, 0) != 0) { + fprintf(stderr, "error running menu callback: %s\n", lua_tostring(L, -1)); + exit(1); + } +} diff --git a/nageru/theme.h b/nageru/theme.h new file mode 100644 index 0000000..0a23995 --- /dev/null +++ b/nageru/theme.h @@ -0,0 +1,187 @@ +#ifndef _THEME_H +#define _THEME_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bmusb/bmusb.h" +#include "ref_counted_frame.h" +#include "tweaked_inputs.h" + +class CEFCapture; +class FFmpegCapture; +class LiveInputWrapper; +struct InputState; + +namespace movit { +class Effect; +class EffectChain; +class ResourcePool; +} // namespace movit + +class Theme { +public: + Theme(const std::string &filename, const std::vector &search_dirs, movit::ResourcePool *resource_pool, unsigned num_cards); + ~Theme(); + + struct Chain { + movit::EffectChain *chain; + std::function setup_chain; + + // FRAME_HISTORY frames for each input, in order. Will contain duplicates + // for non-interlaced inputs. + std::vector input_frames; + }; + + Chain get_chain(unsigned num, float t, unsigned width, unsigned height, InputState input_state); + + int get_num_channels() const { return num_channels; } + int map_signal(int signal_num); + void set_signal_mapping(int signal_num, int card_num); + std::string get_channel_name(unsigned channel); + int get_channel_signal(unsigned channel); + bool get_supports_set_wb(unsigned channel); + void set_wb(unsigned channel, double r, double g, double b); + std::string get_channel_color(unsigned channel); + + std::vector get_transition_names(float t); + + void transition_clicked(int transition_num, float t); + void channel_clicked(int preview_num); + + movit::ResourcePool *get_resource_pool() const { return resource_pool; } + + // Should be called as part of VideoInput.new() only. + void register_video_input(FFmpegCapture *capture) + { + video_inputs.push_back(capture); + } + + std::vector get_video_inputs() const + { + return video_inputs; + } + +#ifdef HAVE_CEF + // Should be called as part of HTMLInput.new() only. + void register_html_input(CEFCapture *capture) + { + html_inputs.push_back(capture); + } + + std::vector get_html_inputs() const + { + return html_inputs; + } +#endif + + void register_video_signal_connection(movit::EffectChain *chain, LiveInputWrapper *live_input, FFmpegCapture *capture) + { + video_signal_connections[chain].emplace_back(VideoSignalConnection { live_input, capture }); + } + +#ifdef HAVE_CEF + void register_html_signal_connection(movit::EffectChain *chain, LiveInputWrapper *live_input, CEFCapture *capture) + { + html_signal_connections[chain].emplace_back(CEFSignalConnection { live_input, capture }); + } +#endif + + struct MenuEntry { + std::string text; + int lua_ref; + }; + std::vector get_theme_menu() { return theme_menu; } // Can be empty for no menu. + void theme_menu_entry_clicked(int lua_ref); + + // Will be invoked every time the theme sets a new menu. + // Is not invoked for a menu that exists at the time of the callback. + void set_theme_menu_callback(std::function callback) + { + theme_menu_callback = callback; + } + +private: + void register_constants(); + void register_class(const char *class_name, const luaL_Reg *funcs); + int set_theme_menu(lua_State *L); + + std::string theme_path; + + std::mutex m; + lua_State *L; // Protected by . + const InputState *input_state = nullptr; // Protected by . Only set temporarily, during chain setup. + movit::ResourcePool *resource_pool; + int num_channels; + unsigned num_cards; + + std::mutex map_m; + std::map signal_to_card_mapping; // Protected by . + + std::vector video_inputs; + struct VideoSignalConnection { + LiveInputWrapper *wrapper; + FFmpegCapture *source; + }; + std::unordered_map> + video_signal_connections; +#ifdef HAVE_CEF + std::vector html_inputs; + struct CEFSignalConnection { + LiveInputWrapper *wrapper; + CEFCapture *source; + }; + std::unordered_map> + html_signal_connections; +#endif + + std::vector theme_menu; + std::function theme_menu_callback; + + friend class LiveInputWrapper; + friend int ThemeMenu_set(lua_State *L); +}; + +// LiveInputWrapper is a facade on top of an YCbCrInput, exposed to +// the Lua code. It contains a function (connect_signal()) intended +// to be called during chain setup, that picks out the current frame +// (in the form of a set of textures) from the input state given by +// the mixer, and communicates that state over to the actual YCbCrInput. +class LiveInputWrapper { +public: + // Note: is irrelevant for PixelFormat_8BitBGRA. + LiveInputWrapper(Theme *theme, movit::EffectChain *chain, bmusb::PixelFormat pixel_format, bool override_bounce, bool deinterlace, bool user_connectable); + + bool connect_signal(int signal_num); // Must be called with the theme's lock held, since it accesses theme->input_state. Returns false on error. + void connect_signal_raw(int signal_num, const InputState &input_state); + movit::Effect *get_effect() const + { + if (deinterlace) { + return deinterlace_effect; + } else if (pixel_format == bmusb::PixelFormat_8BitBGRA) { + return rgba_inputs[0]; + } else { + return ycbcr_inputs[0]; + } + } + +private: + Theme *theme; // Not owned by us. + bmusb::PixelFormat pixel_format; + movit::YCbCrFormat input_ycbcr_format; + std::vector ycbcr_inputs; // Multiple ones if deinterlacing. Owned by the chain. + std::vector rgba_inputs; // Multiple ones if deinterlacing. Owned by the chain. + movit::Effect *deinterlace_effect = nullptr; // Owned by the chain. + bool deinterlace; + bool user_connectable; +}; + +#endif // !defined(_THEME_H) diff --git a/nageru/theme.lua b/nageru/theme.lua new file mode 100644 index 0000000..401cbd9 --- /dev/null +++ b/nageru/theme.lua @@ -0,0 +1,883 @@ +-- The theme is what decides what's actually shown on screen, what kind of +-- transitions are available (if any), and what kind of inputs there are, +-- if any. In general, it drives the entire display logic by creating Movit +-- chains, setting their parameters and then deciding which to show when. +-- +-- Themes are written in Lua, which reflects a simplified form of the Movit API +-- where all the low-level details (such as texture formats) are handled by the +-- C++ side and you generally just build chains. + +local state = { + transition_start = -2.0, + transition_end = -1.0, + transition_type = 0, + transition_src_signal = 0, + transition_dst_signal = 0, + + neutral_colors = { + {0.5, 0.5, 0.5}, -- Input 0. + {0.5, 0.5, 0.5} -- Input 1. + }, + + live_signal_num = 0, + preview_signal_num = 1 +} + +-- Valid values for live_signal_num and preview_signal_num. +local INPUT0_SIGNAL_NUM = 0 +local INPUT1_SIGNAL_NUM = 1 +local SBS_SIGNAL_NUM = 2 +local STATIC_SIGNAL_NUM = 3 + +-- Valid values for transition_type. (Cuts are done directly, so they need no entry.) +local NO_TRANSITION = 0 +local ZOOM_TRANSITION = 1 -- Also for slides. +local FADE_TRANSITION = 2 + +-- Last width/height/frame rate for each channel, if we have it. +-- Note that unlike the values we get from Nageru, the resolution is per +-- frame and not per field, since we deinterlace. +local last_resolution = {} + +-- Utility function to help creating many similar chains that can differ +-- in a free set of chosen parameters. +function make_cartesian_product(parms, callback) + return make_cartesian_product_internal(parms, callback, 1, {}) +end + +function make_cartesian_product_internal(parms, callback, index, args) + if index > #parms then + return callback(unpack(args)) + end + local ret = {} + for _, value in ipairs(parms[index]) do + args[index] = value + ret[value] = make_cartesian_product_internal(parms, callback, index + 1, args) + end + return ret +end + +function make_sbs_input(chain, signal, deint, hq) + local input = chain:add_live_input(not deint, deint) -- Override bounce only if not deinterlacing. + input:connect_signal(signal) + + local resample_effect = nil + local resize_effect = nil + if (hq) then + resample_effect = chain:add_effect(ResampleEffect.new()) + else + resize_effect = chain:add_effect(ResizeEffect.new()) + end + local wb_effect = chain:add_effect(WhiteBalanceEffect.new()) + + local padding_effect = chain:add_effect(IntegralPaddingEffect.new()) + + return { + input = input, + wb_effect = wb_effect, + resample_effect = resample_effect, + resize_effect = resize_effect, + padding_effect = padding_effect + } +end + +-- The main live chain. +function make_sbs_chain(input0_type, input1_type, hq) + local chain = EffectChain.new(16, 9) + + local input0 = make_sbs_input(chain, INPUT0_SIGNAL_NUM, input0_type == "livedeint", hq) + local input1 = make_sbs_input(chain, INPUT1_SIGNAL_NUM, input1_type == "livedeint", hq) + + input0.padding_effect:set_vec4("border_color", 0.0, 0.0, 0.0, 1.0) + input1.padding_effect:set_vec4("border_color", 0.0, 0.0, 0.0, 0.0) + + chain:add_effect(OverlayEffect.new(), input0.padding_effect, input1.padding_effect) + chain:finalize(hq) + + return { + chain = chain, + input0 = input0, + input1 = input1 + } +end + +-- Make all possible combinations of side-by-side chains. +local sbs_chains = make_cartesian_product({ + {"live", "livedeint"}, -- input0_type + {"live", "livedeint"}, -- input1_type + {true, false} -- hq +}, function(input0_type, input1_type, hq) + return make_sbs_chain(input0_type, input1_type, hq) +end) + +function make_fade_input(chain, signal, live, deint, scale) + local input, wb_effect, resample_effect, last + if live then + input = chain:add_live_input(false, deint) + input:connect_signal(signal) + last = input + else + input = chain:add_effect(ImageInput.new("bg.jpeg")) + last = input + end + + -- If we cared about this for the non-main inputs, we would have + -- checked hq here and invoked ResizeEffect instead. + if scale then + resample_effect = chain:add_effect(ResampleEffect.new()) + last = resample_effect + end + + -- Make sure to put the white balance after the scaling (usually more efficient). + if live then + wb_effect = chain:add_effect(WhiteBalanceEffect.new()) + last = wb_effect + end + + return { + input = input, + wb_effect = wb_effect, + resample_effect = resample_effect, + last = last + } +end + +-- A chain to fade between two inputs, of which either can be a picture +-- or a live input. In practice only used live, but we still support the +-- hq parameter. +function make_fade_chain(input0_live, input0_deint, input0_scale, input1_live, input1_deint, input1_scale, hq) + local chain = EffectChain.new(16, 9) + + local input0 = make_fade_input(chain, INPUT0_SIGNAL_NUM, input0_live, input0_deint, input0_scale) + local input1 = make_fade_input(chain, INPUT1_SIGNAL_NUM, input1_live, input1_deint, input1_scale) + + local mix_effect = chain:add_effect(MixEffect.new(), input0.last, input1.last) + chain:finalize(hq) + + return { + chain = chain, + input0 = input0, + input1 = input1, + mix_effect = mix_effect + } +end + +-- Chains to fade between two inputs, in various configurations. +local fade_chains = make_cartesian_product({ + {"static", "live", "livedeint"}, -- input0_type + {true, false}, -- input0_scale + {"static", "live", "livedeint"}, -- input1_type + {true, false}, -- input1_scale + {true} -- hq +}, function(input0_type, input0_scale, input1_type, input1_scale, hq) + local input0_live = (input0_type ~= "static") + local input1_live = (input1_type ~= "static") + local input0_deint = (input0_type == "livedeint") + local input1_deint = (input1_type == "livedeint") + return make_fade_chain(input0_live, input0_deint, input0_scale, input1_live, input1_deint, input1_scale, hq) +end) + +-- A chain to show a single input on screen. +function make_simple_chain(input_deint, input_scale, hq) + local chain = EffectChain.new(16, 9) + + local input = chain:add_live_input(false, input_deint) + input:connect_signal(0) -- First input card. Can be changed whenever you want. + + local resample_effect, resize_effect + if input_scale then + if hq then + resample_effect = chain:add_effect(ResampleEffect.new()) + else + resize_effect = chain:add_effect(ResizeEffect.new()) + end + end + + local wb_effect = chain:add_effect(WhiteBalanceEffect.new()) + chain:finalize(hq) + + return { + chain = chain, + input = input, + wb_effect = wb_effect, + resample_effect = resample_effect, + resize_effect = resize_effect + } +end + +-- Make all possible combinations of single-input chains. +local simple_chains = make_cartesian_product({ + {"live", "livedeint"}, -- input_type + {true, false}, -- input_scale + {true, false} -- hq +}, function(input_type, input_scale, hq) + local input_deint = (input_type == "livedeint") + return make_simple_chain(input_deint, input_scale, hq) +end) + +-- A chain to show a single static picture on screen (HQ version). +local static_chain_hq = EffectChain.new(16, 9) +local static_chain_hq_input = static_chain_hq:add_effect(ImageInput.new("bg.jpeg")) +static_chain_hq:finalize(true) + +-- A chain to show a single static picture on screen (LQ version). +local static_chain_lq = EffectChain.new(16, 9) +local static_chain_lq_input = static_chain_lq:add_effect(ImageInput.new("bg.jpeg")) +static_chain_lq:finalize(false) + +-- Used for indexing into the tables of chains. +function get_input_type(signals, signal_num) + if signal_num == STATIC_SIGNAL_NUM then + return "static" + elseif signals:get_interlaced(signal_num) then + return "livedeint" + else + return "live" + end +end + +function needs_scale(signals, signal_num, width, height) + if signal_num == STATIC_SIGNAL_NUM then + -- We assume this is already correctly scaled at load time. + return false + end + assert(is_plain_signal(signal_num)) + return (signals:get_width(signal_num) ~= width or signals:get_height(signal_num) ~= height) +end + +function set_scale_parameters_if_needed(chain_or_input, width, height) + if chain_or_input.resample_effect then + chain_or_input.resample_effect:set_int("width", width) + chain_or_input.resample_effect:set_int("height", height) + elseif chain_or_input.resize_effect then + chain_or_input.resize_effect:set_int("width", width) + chain_or_input.resize_effect:set_int("height", height) + end +end + +-- API ENTRY POINT +-- Returns the number of outputs in addition to the live (0) and preview (1). +-- Called only once, at the start of the program. +function num_channels() + return 4 +end + +function is_plain_signal(num) + return num == INPUT0_SIGNAL_NUM or num == INPUT1_SIGNAL_NUM +end + +-- Helper function to write e.g. “720p60”. The difference between this +-- and get_channel_resolution_raw() is that this one also can say that +-- there's no signal. +function get_channel_resolution(signal_num) + local res = last_resolution[signal_num] + if (not res) or not res.is_connected then + return "disconnected" + end + if res.height <= 0 then + return "no signal" + end + if not res.has_signal then + if res.height == 525 then + -- Special mode for the USB3 cards. + return "no signal" + end + return get_channel_resolution_raw(res) .. ", no signal" + else + return get_channel_resolution_raw(res) + end +end + +-- Helper function to write e.g. “60” or “59.94”. +function get_frame_rate(res) + local nom = res.frame_rate_nom + local den = res.frame_rate_den + if nom % den == 0 then + return nom / den + else + return string.format("%.2f", nom / den) + end +end + +-- Helper function to write e.g. “720p60”. +function get_channel_resolution_raw(res) + if res.interlaced then + return res.height .. "i" .. get_frame_rate(res) + else + return res.height .. "p" .. get_frame_rate(res) + end +end + +-- API ENTRY POINT +-- Returns the name for each additional channel (starting from 2). +-- Called at the start of the program, and then each frame for live +-- channels in case they change resolution. +function channel_name(channel) + local signal_num = channel - 2 + if is_plain_signal(signal_num) then + return "Input " .. (signal_num + 1) .. " (" .. get_channel_resolution(signal_num) .. ")" + elseif signal_num == SBS_SIGNAL_NUM then + return "Side-by-side" + elseif signal_num == STATIC_SIGNAL_NUM then + return "Static picture" + end +end + +-- API ENTRY POINT +-- Returns, given a channel number, which signal it corresponds to (starting from 0). +-- Should return -1 if the channel does not correspond to a simple signal +-- (one connected to a capture card, or a video input). The information is used for +-- whether right-click on the channel should bring up a context menu or not, +-- typically containing an input selector, resolution menu etc. +-- +-- Called once for each channel, at the start of the program. +-- Will never be called for live (0) or preview (1). +function channel_signal(channel) + if channel == 2 then + return 0 + elseif channel == 3 then + return 1 + else + return -1 + end +end + +-- API ENTRY POINT +-- Called every frame. Returns the color (if any) to paint around the given +-- channel. Returns a CSS color (typically to mark live and preview signals); +-- "transparent" is allowed. +-- Will never be called for live (0) or preview (1). +function channel_color(channel) + if state.transition_type ~= NO_TRANSITION then + if channel_involved_in(channel, state.transition_src_signal) or + channel_involved_in(channel, state.transition_dst_signal) then + return "#f00" + end + else + if channel_involved_in(channel, state.live_signal_num) then + return "#f00" + end + end + if channel_involved_in(channel, state.preview_signal_num) then + return "#0f0" + end + return "transparent" +end + +function channel_involved_in(channel, signal_num) + if is_plain_signal(signal_num) then + return channel == (signal_num + 2) + end + if signal_num == SBS_SIGNAL_NUM then + return (channel == 2 or channel == 3) + end + if signal_num == STATIC_SIGNAL_NUM then + return (channel == 5) + end + return false +end + +-- API ENTRY POINT +-- Returns if a given channel supports setting white balance (starting from 2). +-- Called only once for each channel, at the start of the program. +function supports_set_wb(channel) + return is_plain_signal(channel - 2) +end + +-- API ENTRY POINT +-- Gets called with a new gray point when the white balance is changing. +-- The color is in linear light (not sRGB gamma). +function set_wb(channel, red, green, blue) + if is_plain_signal(channel - 2) then + state.neutral_colors[channel - 2 + 1] = { red, green, blue } + end +end + +function finish_transitions(t) + if state.transition_type ~= NO_TRANSITION and t >= state.transition_end then + state.live_signal_num = state.transition_dst_signal + state.transition_type = NO_TRANSITION + end +end + +function in_transition(t) + return t >= state.transition_start and t <= state.transition_end +end + +-- API ENTRY POINT +-- Called every frame. +function get_transitions(t) + if in_transition(t) then + -- Transition already in progress, the only thing we can do is really + -- cut to the preview. (TODO: Make an “abort” and/or “finish”, too?) + return {"Cut"} + end + + finish_transitions(t) + + if state.live_signal_num == state.preview_signal_num then + -- No transitions possible. + return {} + end + + if (is_plain_signal(state.live_signal_num) or state.live_signal_num == STATIC_SIGNAL_NUM) and + (is_plain_signal(state.preview_signal_num) or state.preview_signal_num == STATIC_SIGNAL_NUM) then + return {"Cut", "", "Fade"} + end + + -- Various zooms. + if state.live_signal_num == SBS_SIGNAL_NUM and is_plain_signal(state.preview_signal_num) then + return {"Cut", "Zoom in"} + elseif is_plain_signal(state.live_signal_num) and state.preview_signal_num == SBS_SIGNAL_NUM then + return {"Cut", "Zoom out"} + end + + return {"Cut"} +end + +function swap_preview_live() + local temp = state.live_signal_num + state.live_signal_num = state.preview_signal_num + state.preview_signal_num = temp +end + +function start_transition(type_, t, duration) + state.transition_start = t + state.transition_end = t + duration + state.transition_type = type_ + state.transition_src_signal = state.live_signal_num + state.transition_dst_signal = state.preview_signal_num + swap_preview_live() +end + +-- API ENTRY POINT +-- Called when the user clicks a transition button. +function transition_clicked(num, t) + if num == 0 then + -- Cut. + if in_transition(t) then + -- Ongoing transition; finish it immediately before the cut. + finish_transitions(state.transition_end) + end + + swap_preview_live() + elseif num == 1 then + -- Zoom. + finish_transitions(t) + + if state.live_signal_num == state.preview_signal_num then + -- Nothing to do. + return + end + + if is_plain_signal(state.live_signal_num) and is_plain_signal(state.preview_signal_num) then + -- We can't zoom between these. Just make a cut. + io.write("Cutting from " .. state.live_signal_num .. " to " .. state.live_signal_num .. "\n") + swap_preview_live() + return + end + + if (state.live_signal_num == SBS_SIGNAL_NUM and is_plain_signal(state.preview_signal_num)) or + (state.preview_signal_num == SBS_SIGNAL_NUM and is_plain_signal(state.live_signal_num)) then + start_transition(ZOOM_TRANSITION, t, 1.0) + end + elseif num == 2 then + finish_transitions(t) + + -- Fade. + if (state.live_signal_num ~= state.preview_signal_num) and + (is_plain_signal(state.live_signal_num) or + state.live_signal_num == STATIC_SIGNAL_NUM) and + (is_plain_signal(state.preview_signal_num) or + state.preview_signal_num == STATIC_SIGNAL_NUM) then + start_transition(FADE_TRANSITION, t, 1.0) + else + -- Fades involving SBS are ignored (we have no chain for it). + end + end +end + +-- API ENTRY POINT +function channel_clicked(num) + state.preview_signal_num = num +end + +function get_fade_chain(state, signals, t, width, height, input_resolution) + local input0_type = get_input_type(signals, state.transition_src_signal) + local input0_scale = needs_scale(signals, state.transition_src_signal, width, height) + local input1_type = get_input_type(signals, state.transition_dst_signal) + local input1_scale = needs_scale(signals, state.transition_dst_signal, width, height) + local chain = fade_chains[input0_type][input0_scale][input1_type][input1_scale][true] + local prepare = function() + if input0_type == "live" or input0_type == "livedeint" then + chain.input0.input:connect_signal(state.transition_src_signal) + set_neutral_color_from_signal(state, chain.input0.wb_effect, state.transition_src_signal) + end + set_scale_parameters_if_needed(chain.input0, width, height) + if input1_type == "live" or input1_type == "livedeint" then + chain.input1.input:connect_signal(state.transition_dst_signal) + set_neutral_color_from_signal(state, chain.input1.wb_effect, state.transition_dst_signal) + end + set_scale_parameters_if_needed(chain.input1, width, height) + local tt = calc_fade_progress(t, state.transition_start, state.transition_end) + + chain.mix_effect:set_float("strength_first", 1.0 - tt) + chain.mix_effect:set_float("strength_second", tt) + end + return chain.chain, prepare +end + +-- SBS code (live_signal_num == SBS_SIGNAL_NUM, or in a transition to/from it). +function get_sbs_chain(signals, t, width, height, input_resolution) + local input0_type = get_input_type(signals, INPUT0_SIGNAL_NUM) + local input1_type = get_input_type(signals, INPUT1_SIGNAL_NUM) + return sbs_chains[input0_type][input1_type][true] +end + +-- API ENTRY POINT +-- Called every frame. Get the chain for displaying at input , +-- where 0 is live, 1 is preview, 2 is the first channel to display +-- in the bottom bar, and so on up to num_channels()+1. t is the +-- current time in seconds. width and height are the dimensions of +-- the output, although you can ignore them if you don't need them +-- (they're useful if you want to e.g. know what to resample by). +-- +-- is basically an exposed InputState, which you can use to +-- query for information about the signals at the point of the current +-- frame. In particular, you can call get_width() and get_height() +-- for any signal number, and use that to e.g. assist in chain selection. +-- +-- You should return two objects; the chain itself, and then a +-- function (taking no parameters) that is run just before rendering. +-- The function needs to call connect_signal on any inputs, so that +-- it gets updated video data for the given frame. (You are allowed +-- to switch which input your input is getting from between frames, +-- but not calling connect_signal results in undefined behavior.) +-- If you want to change any parameters in the chain, this is also +-- the right place. +-- +-- NOTE: The chain returned must be finalized with the Y'CbCr flag +-- if and only if num==0. +function get_chain(num, t, width, height, signals) + local input_resolution = {} + for signal_num=0,1 do + local res = { + width = signals:get_width(signal_num), + height = signals:get_height(signal_num), + interlaced = signals:get_interlaced(signal_num), + is_connected = signals:get_is_connected(signal_num), + has_signal = signals:get_has_signal(signal_num), + frame_rate_nom = signals:get_frame_rate_nom(signal_num), + frame_rate_den = signals:get_frame_rate_den(signal_num) + } + + if res.interlaced then + -- Convert height from frame height to field height. + -- (Needed for e.g. place_rectangle.) + res.height = res.height * 2 + + -- Show field rate instead of frame rate; really for cosmetics only + -- (and actually contrary to EBU recommendations, although in line + -- with typical user expectations). + res.frame_rate_nom = res.frame_rate_nom * 2 + end + + input_resolution[signal_num] = res + end + last_resolution = input_resolution + + -- Make a (semi-shallow) copy of the current state, so that the returned prepare function + -- is unaffected by state changes made by the UI before it is rendered. + local state_copy = {} + for key, value in pairs(state) do + state_copy[key] = value + end + state_copy.neutral_colors = { unpack(state.neutral_colors) } + + if num == 0 then -- Live. + finish_transitions(t) + if state.transition_type == ZOOM_TRANSITION then + -- Transition in or out of SBS. + local chain = get_sbs_chain(signals, t, width, height, input_resolution) + local prepare = function() + prepare_sbs_chain(state_copy, chain, calc_zoom_progress(state_copy, t), state_copy.transition_type, state_copy.transition_src_signal, state_copy.transition_dst_signal, width, height, input_resolution) + end + return chain.chain, prepare + elseif state.transition_type == NO_TRANSITION and state.live_signal_num == SBS_SIGNAL_NUM then + -- Static SBS view. + local chain = get_sbs_chain(signals, t, width, height, input_resolution) + local prepare = function() + prepare_sbs_chain(state_copy, chain, 0.0, NO_TRANSITION, 0, SBS_SIGNAL_NUM, width, height, input_resolution) + end + return chain.chain, prepare + elseif state.transition_type == FADE_TRANSITION then + return get_fade_chain(state_copy, signals, t, width, height, input_resolution) + elseif is_plain_signal(state.live_signal_num) then + local input_type = get_input_type(signals, state.live_signal_num) + local input_scale = needs_scale(signals, state.live_signal_num, width, height) + local chain = simple_chains[input_type][input_scale][true] + local prepare = function() + chain.input:connect_signal(state_copy.live_signal_num) + set_scale_parameters_if_needed(chain, width, height) + set_neutral_color_from_signal(state_copy, chain.wb_effect, state_copy.live_signal_num) + end + return chain.chain, prepare + elseif state.live_signal_num == STATIC_SIGNAL_NUM then -- Static picture. + local prepare = function() + end + return static_chain_hq, prepare + else + assert(false) + end + end + if num == 1 then -- Preview. + num = state.preview_signal_num + 2 + end + + -- Individual preview inputs. + if is_plain_signal(num - 2) then + local signal_num = num - 2 + local input_type = get_input_type(signals, signal_num) + local input_scale = needs_scale(signals, signal_num, width, height) + local chain = simple_chains[input_type][input_scale][false] + local prepare = function() + chain.input:connect_signal(signal_num) + set_scale_parameters_if_needed(chain, width, height) + set_neutral_color(chain.wb_effect, state_copy.neutral_colors[signal_num + 1]) + end + return chain.chain, prepare + end + if num == SBS_SIGNAL_NUM + 2 then + local input0_type = get_input_type(signals, INPUT0_SIGNAL_NUM) + local input1_type = get_input_type(signals, INPUT1_SIGNAL_NUM) + local chain = sbs_chains[input0_type][input1_type][false] + local prepare = function() + prepare_sbs_chain(state_copy, chain, 0.0, NO_TRANSITION, 0, SBS_SIGNAL_NUM, width, height, input_resolution) + end + return chain.chain, prepare + end + if num == STATIC_SIGNAL_NUM + 2 then + local prepare = function() + end + return static_chain_lq, prepare + end +end + +function place_rectangle(resample_effect, resize_effect, padding_effect, x0, y0, x1, y1, screen_width, screen_height, input_width, input_height) + local srcx0 = 0.0 + local srcx1 = 1.0 + local srcy0 = 0.0 + local srcy1 = 1.0 + + padding_effect:set_int("width", screen_width) + padding_effect:set_int("height", screen_height) + + -- Cull. + if x0 > screen_width or x1 < 0.0 or y0 > screen_height or y1 < 0.0 then + if resample_effect ~= nil then + resample_effect:set_int("width", 1) + resample_effect:set_int("height", 1) + resample_effect:set_float("zoom_x", screen_width) + resample_effect:set_float("zoom_y", screen_height) + else + resize_effect:set_int("width", 1) + resize_effect:set_int("height", 1) + end + padding_effect:set_int("left", screen_width + 100) + padding_effect:set_int("top", screen_height + 100) + return + end + + -- Clip. + if x0 < 0 then + srcx0 = -x0 / (x1 - x0) + x0 = 0 + end + if y0 < 0 then + srcy0 = -y0 / (y1 - y0) + y0 = 0 + end + if x1 > screen_width then + srcx1 = (screen_width - x0) / (x1 - x0) + x1 = screen_width + end + if y1 > screen_height then + srcy1 = (screen_height - y0) / (y1 - y0) + y1 = screen_height + end + + if resample_effect ~= nil then + -- High-quality resampling. + local x_subpixel_offset = x0 - math.floor(x0) + local y_subpixel_offset = y0 - math.floor(y0) + + -- Resampling must be to an integral number of pixels. Round up, + -- and then add an extra pixel so we have some leeway for the border. + local width = math.ceil(x1 - x0) + 1 + local height = math.ceil(y1 - y0) + 1 + resample_effect:set_int("width", width) + resample_effect:set_int("height", height) + + -- Correct the discrepancy with zoom. (This will leave a small + -- excess edge of pixels and subpixels, which we'll correct for soon.) + local zoom_x = (x1 - x0) / (width * (srcx1 - srcx0)) + local zoom_y = (y1 - y0) / (height * (srcy1 - srcy0)) + resample_effect:set_float("zoom_x", zoom_x) + resample_effect:set_float("zoom_y", zoom_y) + resample_effect:set_float("zoom_center_x", 0.0) + resample_effect:set_float("zoom_center_y", 0.0) + + -- Padding must also be to a whole-pixel offset. + padding_effect:set_int("left", math.floor(x0)) + padding_effect:set_int("top", math.floor(y0)) + + -- Correct _that_ discrepancy by subpixel offset in the resampling. + resample_effect:set_float("left", srcx0 * input_width - x_subpixel_offset / zoom_x) + resample_effect:set_float("top", srcy0 * input_height - y_subpixel_offset / zoom_y) + + -- Finally, adjust the border so it is exactly where we want it. + padding_effect:set_float("border_offset_left", x_subpixel_offset) + padding_effect:set_float("border_offset_right", x1 - (math.floor(x0) + width)) + padding_effect:set_float("border_offset_top", y_subpixel_offset) + padding_effect:set_float("border_offset_bottom", y1 - (math.floor(y0) + height)) + else + -- Lower-quality simple resizing. + local width = round(x1 - x0) + local height = round(y1 - y0) + resize_effect:set_int("width", width) + resize_effect:set_int("height", height) + + -- Padding must also be to a whole-pixel offset. + padding_effect:set_int("left", math.floor(x0)) + padding_effect:set_int("top", math.floor(y0)) + end +end + +-- This is broken, of course (even for positive numbers), but Lua doesn't give us access to real rounding. +function round(x) + return math.floor(x + 0.5) +end + +function lerp(a, b, t) + return a + (b - a) * t +end + +function lerp_pos(a, b, t) + return { + x0 = lerp(a.x0, b.x0, t), + y0 = lerp(a.y0, b.y0, t), + x1 = lerp(a.x1, b.x1, t), + y1 = lerp(a.y1, b.y1, t) + } +end + +function pos_from_top_left(x, y, width, height, screen_width, screen_height) + local xs = screen_width / 1280.0 + local ys = screen_height / 720.0 + return { + x0 = round(xs * x), + y0 = round(ys * y), + x1 = round(xs * (x + width)), + y1 = round(ys * (y + height)) + } +end + +function prepare_sbs_chain(state, chain, t, transition_type, src_signal, dst_signal, screen_width, screen_height, input_resolution) + chain.input0.input:connect_signal(0) + chain.input1.input:connect_signal(1) + set_neutral_color(chain.input0.wb_effect, state.neutral_colors[1]) + set_neutral_color(chain.input1.wb_effect, state.neutral_colors[2]) + + -- First input is positioned (16,48) from top-left. + -- Second input is positioned (16,48) from the bottom-right. + local pos0 = pos_from_top_left(16, 48, 848, 477, screen_width, screen_height) + local pos1 = pos_from_top_left(1280 - 384 - 16, 720 - 216 - 48, 384, 216, screen_width, screen_height) + + local pos_fs = { x0 = 0, y0 = 0, x1 = screen_width, y1 = screen_height } + local affine_param + if transition_type == NO_TRANSITION then + -- Static SBS view. + affine_param = { sx = 1.0, sy = 1.0, tx = 0.0, ty = 0.0 } -- Identity. + else + -- Zooming to/from SBS view into or out of a single view. + assert(transition_type == ZOOM_TRANSITION) + local signal, real_t + if src_signal == SBS_SIGNAL_NUM then + signal = dst_signal + real_t = t + else + assert(dst_signal == SBS_SIGNAL_NUM) + signal = src_signal + real_t = 1.0 - t + end + + if signal == INPUT0_SIGNAL_NUM then + affine_param = find_affine_param(pos0, lerp_pos(pos0, pos_fs, real_t)) + elseif signal == INPUT1_SIGNAL_NUM then + affine_param = find_affine_param(pos1, lerp_pos(pos1, pos_fs, real_t)) + end + end + + -- NOTE: input_resolution is not 1-indexed, unlike usual Lua arrays. + place_rectangle_with_affine(chain.input0.resample_effect, chain.input0.resize_effect, chain.input0.padding_effect, pos0, affine_param, screen_width, screen_height, input_resolution[0].width, input_resolution[0].height) + place_rectangle_with_affine(chain.input1.resample_effect, chain.input1.resize_effect, chain.input1.padding_effect, pos1, affine_param, screen_width, screen_height, input_resolution[1].width, input_resolution[1].height) +end + +-- Find the transformation that changes the first rectangle to the second one. +function find_affine_param(a, b) + local sx = (b.x1 - b.x0) / (a.x1 - a.x0) + local sy = (b.y1 - b.y0) / (a.y1 - a.y0) + return { + sx = sx, + sy = sy, + tx = b.x0 - a.x0 * sx, + ty = b.y0 - a.y0 * sy + } +end + +function place_rectangle_with_affine(resample_effect, resize_effect, padding_effect, pos, aff, screen_width, screen_height, input_width, input_height) + local x0 = pos.x0 * aff.sx + aff.tx + local x1 = pos.x1 * aff.sx + aff.tx + local y0 = pos.y0 * aff.sy + aff.ty + local y1 = pos.y1 * aff.sy + aff.ty + + place_rectangle(resample_effect, resize_effect, padding_effect, x0, y0, x1, y1, screen_width, screen_height, input_width, input_height) +end + +function set_neutral_color(effect, color) + effect:set_vec3("neutral_color", color[1], color[2], color[3]) +end + +function set_neutral_color_from_signal(state, effect, signal) + if is_plain_signal(signal) then + set_neutral_color(effect, state.neutral_colors[signal - INPUT0_SIGNAL_NUM + 1]) + end +end + +function calc_zoom_progress(state, t) + if t < state.transition_start then + return 0.0 + elseif t > state.transition_end then + return 1.0 + else + local tt = (t - state.transition_start) / (state.transition_end - state.transition_start) + -- Smooth it a bit. + return math.sin(tt * 3.14159265358 * 0.5) + end +end + +function calc_fade_progress(t, transition_start, transition_end) + local tt = (t - transition_start) / (transition_end - transition_start) + if tt < 0.0 then + return 0.0 + elseif tt > 1.0 then + return 1.0 + end + + -- Make the fade look maybe a tad more natural, by pumping it + -- through a sigmoid function. + tt = 10.0 * tt - 5.0 + tt = 1.0 / (1.0 + math.exp(-tt)) + + return tt +end diff --git a/nageru/timebase.h b/nageru/timebase.h new file mode 100644 index 0000000..dbc4402 --- /dev/null +++ b/nageru/timebase.h @@ -0,0 +1,25 @@ +#ifndef _TIMEBASE_H +#define _TIMEBASE_H 1 + +// Common timebase that allows us to represent one frame exactly in all the +// relevant frame rates: +// +// Timebase: 1/120000 +// Frame at 50fps: 2400/120000 +// Frame at 60fps: 2000/120000 +// Frame at 59.94fps: 2002/120000 +// Frame at 23.976fps: 5005/120000 +// +// If we also wanted to represent one sample at 48000 Hz, we'd need +// to go to 300000. Also supporting one sample at 44100 Hz would mean +// going to 44100000; probably a bit excessive. +#define TIMEBASE 120000 + +// Some muxes, like MP4 (or at least avformat's implementation of it), +// are not too fond of values above 2^31. At timebase 120000, that's only +// about five hours or so, so we define a coarser timebase that doesn't +// get 59.94 precisely (so there will be a marginal amount of pts jitter), +// but can do at least 50 and 60 precisely, and months of streaming. +#define COARSE_TIMEBASE 300 + +#endif // !defined(_TIMEBASE_H) diff --git a/nageru/timecode_renderer.cpp b/nageru/timecode_renderer.cpp new file mode 100644 index 0000000..a923acd --- /dev/null +++ b/nageru/timecode_renderer.cpp @@ -0,0 +1,214 @@ +#include "timecode_renderer.h" + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include "flags.h" + +using namespace std; +using namespace movit; + +TimecodeRenderer::TimecodeRenderer(movit::ResourcePool *resource_pool, unsigned display_width, unsigned display_height) + : resource_pool(resource_pool), display_width(display_width), display_height(display_height), height(28) +{ + string vert_shader = + "#version 130 \n" + " \n" + "in vec2 position; \n" + "in vec2 texcoord; \n" + "out vec2 tc0; \n" + " \n" + "void main() \n" + "{ \n" + " // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is: \n" + " // \n" + " // 2.000 0.000 0.000 -1.000 \n" + " // 0.000 2.000 0.000 -1.000 \n" + " // 0.000 0.000 -2.000 -1.000 \n" + " // 0.000 0.000 0.000 1.000 \n" + " gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0); \n" + " tc0 = texcoord; \n" + "} \n"; + string frag_shader = + "#version 130 \n" + "in vec2 tc0; \n" + "uniform sampler2D tex; \n" + "out vec4 Y, CbCr, YCbCr; \n" + "void main() { \n" + " vec4 gray = texture(tex, tc0); \n"; + if (global_flags.ten_bit_output) { + frag_shader += + " gray.r = gray.r * ((940.0-16.0)/65535.0) + 16.0/65535.0; \n" // Limited-range Y'CbCr. + " CbCr = vec4(512.0/65535.0, 512.0/65535.0, 0.0, 1.0); \n"; + } else { + frag_shader += + " gray.r = gray.r * ((235.0-16.0)/255.0) + 16.0/255.0; \n" // Limited-range Y'CbCr. + " CbCr = vec4(128.0/255.0, 128.0/255.0, 0.0, 1.0); \n"; + } + frag_shader += + " Y = gray.rrra; \n" + " YCbCr = vec4(Y.r, CbCr.r, CbCr.g, CbCr.a); \n" + "} \n"; + + vector frag_shader_outputs; + program_num = resource_pool->compile_glsl_program(vert_shader, frag_shader, frag_shader_outputs); + check_error(); + + texture_sampler_uniform = glGetUniformLocation(program_num, "tex"); + check_error(); + position_attribute_index = glGetAttribLocation(program_num, "position"); + check_error(); + texcoord_attribute_index = glGetAttribLocation(program_num, "texcoord"); + check_error(); + + // Shared between the two. + float vertices[] = { + 0.0f, 2.0f, + 0.0f, 0.0f, + 2.0f, 0.0f + }; + vbo = generate_vbo(2, GL_FLOAT, sizeof(vertices), vertices); + check_error(); + + tex = resource_pool->create_2d_texture(GL_R8, display_width, height); + + image.reset(new QImage(display_width, height, QImage::Format_Grayscale8)); +} + +TimecodeRenderer::~TimecodeRenderer() +{ + resource_pool->release_2d_texture(tex); + check_error(); + resource_pool->release_glsl_program(program_num); + check_error(); + glDeleteBuffers(1, &vbo); + check_error(); +} + +string TimecodeRenderer::get_timecode_text(double pts, unsigned frame_num) +{ + // Find the wall time, and round it to the nearest millisecond. + timeval now; + gettimeofday(&now, nullptr); + time_t unixtime = now.tv_sec; + unsigned msecs = (now.tv_usec + 500) / 1000; + if (msecs >= 1000) { + msecs -= 1000; + ++unixtime; + } + + tm utc_tm; + gmtime_r(&unixtime, &utc_tm); + char clock_text[256]; + strftime(clock_text, sizeof(clock_text), "%Y-%m-%d %H:%M:%S", &utc_tm); + + // Make the stream timecode, rounded to the nearest millisecond. + long stream_time = lrint(pts * 1e3); + assert(stream_time >= 0); + unsigned stream_time_ms = stream_time % 1000; + stream_time /= 1000; + unsigned stream_time_sec = stream_time % 60; + stream_time /= 60; + unsigned stream_time_min = stream_time % 60; + unsigned stream_time_hour = stream_time / 60; + + char timecode_text[512]; + snprintf(timecode_text, sizeof(timecode_text), "Nageru - %s.%03u UTC - Stream time %02u:%02u:%02u.%03u (frame %u)", + clock_text, msecs, stream_time_hour, stream_time_min, stream_time_sec, stream_time_ms, frame_num); + return timecode_text; +} + +void TimecodeRenderer::render_timecode(GLuint fbo, const string &text) +{ + render_string_to_buffer(text); + render_buffer_to_fbo(fbo); +} + +void TimecodeRenderer::render_string_to_buffer(const string &text) +{ + image->fill(0); + QPainter painter(image.get()); + + painter.setPen(Qt::white); + QFont font = painter.font(); + font.setPointSize(16); + painter.setFont(font); + + painter.drawText(QRectF(0, 0, display_width, height), Qt::AlignCenter, QString::fromStdString(text)); +} + +void TimecodeRenderer::render_buffer_to_fbo(GLuint fbo) +{ + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + check_error(); + + GLuint vao; + glGenVertexArrays(1, &vao); + check_error(); + + glBindVertexArray(vao); + check_error(); + + glViewport(0, display_height - height, display_width, height); + check_error(); + + glActiveTexture(GL_TEXTURE0); + check_error(); + glBindTexture(GL_TEXTURE_2D, tex); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, display_width, height, GL_RED, GL_UNSIGNED_BYTE, image->bits()); + check_error(); + + glUseProgram(program_num); + check_error(); + glUniform1i(texture_sampler_uniform, 0); + check_error(); + + glBindBuffer(GL_ARRAY_BUFFER, vbo); + check_error(); + + for (GLint attr_index : { position_attribute_index, texcoord_attribute_index }) { + if (attr_index == -1) continue; + glEnableVertexAttribArray(attr_index); + check_error(); + glVertexAttribPointer(attr_index, 2, GL_FLOAT, GL_FALSE, 0, BUFFER_OFFSET(0)); + check_error(); + } + + glDrawArrays(GL_TRIANGLES, 0, 3); + check_error(); + + for (GLint attr_index : { position_attribute_index, texcoord_attribute_index }) { + if (attr_index == -1) continue; + glDisableVertexAttribArray(attr_index); + check_error(); + } + + glActiveTexture(GL_TEXTURE0); + check_error(); + glUseProgram(0); + check_error(); + + glDeleteVertexArrays(1, &vao); + check_error(); + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + check_error(); +} diff --git a/nageru/timecode_renderer.h b/nageru/timecode_renderer.h new file mode 100644 index 0000000..809a829 --- /dev/null +++ b/nageru/timecode_renderer.h @@ -0,0 +1,48 @@ +#ifndef _TIMECODE_RENDERER_H +#define _TIMECODE_RENDERER_H 1 + +#include +#include + +#include + +// A class to render a simple text string onto the picture using Qt and OpenGL. + +namespace movit { + +class ResourcePool; + +} // namespace movit + +class QImage; + +class TimecodeRenderer { +public: + TimecodeRenderer(movit::ResourcePool *resource_pool, unsigned display_width, unsigned display_height); + ~TimecodeRenderer(); + + // Return a string with the current wall clock time and the + // logical stream time. + static std::string get_timecode_text(double pts, unsigned frame_num); + + // The FBO is assumed to contain three outputs (Y', Cb/Cr and RGBA). + void render_timecode(GLuint fbo, const std::string &text); + +private: + void render_string_to_buffer(const std::string &text); + void render_buffer_to_fbo(GLuint fbo); + + movit::ResourcePool *resource_pool; + unsigned display_width, display_height, height; + + GLuint vbo; // Holds position and texcoord data. + GLuint tex; + //std::unique_ptr pixel_buffer; + std::unique_ptr image; + + GLuint program_num; // Owned by . + GLuint texture_sampler_uniform; + GLuint position_attribute_index, texcoord_attribute_index; +}; + +#endif diff --git a/nageru/tweaked_inputs.cpp b/nageru/tweaked_inputs.cpp new file mode 100644 index 0000000..304c3b4 --- /dev/null +++ b/nageru/tweaked_inputs.cpp @@ -0,0 +1,48 @@ +#include +#include +#include + +#include "tweaked_inputs.h" + +sRGBSwitchingFlatInput::~sRGBSwitchingFlatInput() +{ + if (sampler_obj != 0) { + glDeleteSamplers(1, &sampler_obj); + } +} + +void sRGBSwitchingFlatInput::set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num) +{ + movit::FlatInput::set_gl_state(glsl_program_num, prefix, sampler_num); + texture_unit = *sampler_num - 1; + + if (sampler_obj == 0) { + glGenSamplers(1, &sampler_obj); + check_error(); + glSamplerParameteri(sampler_obj, GL_TEXTURE_MIN_FILTER, needs_mipmaps ? GL_LINEAR_MIPMAP_NEAREST : GL_LINEAR); + check_error(); + glSamplerParameteri(sampler_obj, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glSamplerParameteri(sampler_obj, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error() + // This needs to be done on a sampler and not a texture parameter, + // because the texture could be used from multiple different + // contexts at the same time. This flag is ignored for non-sRGB-uploaded + // textures, so we can set it without checking can_output_linear_gamma(). + if (output_linear_gamma) { + glSamplerParameteri(sampler_obj, GL_TEXTURE_SRGB_DECODE_EXT, GL_DECODE_EXT); + } else { + glSamplerParameteri(sampler_obj, GL_TEXTURE_SRGB_DECODE_EXT, GL_SKIP_DECODE_EXT); + } + check_error(); + } + + glBindSampler(texture_unit, sampler_obj); + check_error(); +} + +void sRGBSwitchingFlatInput::clear_gl_state() +{ + glBindSampler(texture_unit, 0); + check_error(); +} diff --git a/nageru/tweaked_inputs.h b/nageru/tweaked_inputs.h new file mode 100644 index 0000000..0ca13ce --- /dev/null +++ b/nageru/tweaked_inputs.h @@ -0,0 +1,73 @@ +#ifndef _TWEAKED_INPUTS_H +#define _TWEAKED_INPUTS_H 1 + +// Some tweaked variations of Movit inputs. + +#include + +namespace movit { +struct ImageFormat; +struct YCbCrFormat; +} // namespace movit + +class NonBouncingYCbCrInput : public movit::YCbCrInput { +public: + NonBouncingYCbCrInput(const movit::ImageFormat &image_format, + const movit::YCbCrFormat &ycbcr_format, + unsigned width, unsigned height, + movit::YCbCrInputSplitting ycbcr_input_splitting = movit::YCBCR_INPUT_PLANAR) + : movit::YCbCrInput(image_format, ycbcr_format, width, height, ycbcr_input_splitting) {} + + bool override_disable_bounce() const override { return true; } +}; + +// We use FlatInput with RGBA inputs a few places where we can't tell when +// uploading the texture whether it needs to be converted from sRGB to linear +// or not. (FlatInput deals with this if you give it pixels, but we give it +// already uploaded textures.) +// +// If we have GL_EXT_texture_sRGB_decode (very common, as far as I can tell), +// we can just always upload with the sRGB flag turned on, and then turn it off +// if not requested; that's sRGBSwitchingFlatInput. If not, we just need to +// turn off the functionality altogether, which is NonsRGBCapableFlatInput. +// +// If you're using NonsRGBCapableFlatInput, upload with GL_RGBA8. +// If using sRGBSwitchingFlatInput, upload with GL_SRGB8_ALPHA8. + +class NonsRGBCapableFlatInput : public movit::FlatInput { +public: + NonsRGBCapableFlatInput(movit::ImageFormat format, movit::MovitPixelFormat pixel_format, GLenum type, unsigned width, unsigned height) + : movit::FlatInput(format, pixel_format, type, width, height) {} + + bool can_output_linear_gamma() const override { return false; } +}; + +class sRGBSwitchingFlatInput : public movit::FlatInput { +public: + sRGBSwitchingFlatInput(movit::ImageFormat format, movit::MovitPixelFormat pixel_format, GLenum type, unsigned width, unsigned height) + : movit::FlatInput(format, pixel_format, type, width, height) {} + + ~sRGBSwitchingFlatInput(); + void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num) override; + void clear_gl_state() override; + + bool set_int(const std::string &key, int value) override + { + if (key == "output_linear_gamma") { + output_linear_gamma = value; + } + if (key == "needs_mipmaps") { + needs_mipmaps = value; + } + return movit::FlatInput::set_int(key, value); + } + +private: + bool output_linear_gamma = false; + bool needs_mipmaps = false; + GLuint sampler_obj = 0; + GLuint texture_unit; +}; + + +#endif // !defined(_TWEAKED_INPUTS_H) diff --git a/nageru/v210_converter.cpp b/nageru/v210_converter.cpp new file mode 100644 index 0000000..d10920b --- /dev/null +++ b/nageru/v210_converter.cpp @@ -0,0 +1,156 @@ +#include "v210_converter.h" + +#include +#include + +using namespace std; + +v210Converter::~v210Converter() +{ + for (const auto &shader : shaders) { + glDeleteProgram(shader.second.glsl_program_num); + check_error(); + } +} + +bool v210Converter::has_hardware_support() +{ + // We don't have a GLES version of this, although GLSL ES 3.1 supports + // compute shaders. Note that GLSL ES has some extra restrictions, + // like requiring that the images are allocated with glTexStorage*(), + // or that binding= is effectively mandatory. + if (!epoxy_is_desktop_gl()) { + return false; + } + if (epoxy_gl_version() >= 43) { + // Supports compute shaders natively. + return true; + } + return epoxy_has_gl_extension("GL_ARB_compute_shader") && + epoxy_has_gl_extension("GL_ARB_shader_image_load_store"); +} + +void v210Converter::precompile_shader(unsigned width) +{ + unsigned num_local_work_groups = (width + 5) / 6; + if (shaders.count(num_local_work_groups)) { + // Already exists. + return; + } + + char buf[16]; + snprintf(buf, sizeof(buf), "%u", num_local_work_groups); + string shader_src = R"(#version 150 +#extension GL_ARB_compute_shader : enable +#extension GL_ARB_shader_image_load_store : enable +layout(local_size_x = )" + string(buf) + R"() in; +layout(rgb10_a2) uniform restrict readonly image2D inbuf; +layout(rgb10_a2) uniform restrict writeonly image2D outbuf; +uniform int max_cbcr_x; +shared vec2 cbcr[gl_WorkGroupSize.x * 3u]; + +void main() +{ + int xb = int(gl_LocalInvocationID.x); // X block. + int y = int(gl_GlobalInvocationID.y); // Y (actual line). + + // Load our pixel group, containing data for six pixels. + vec3 indata[4]; + for (int i = 0; i < 4; ++i) { + indata[i] = imageLoad(inbuf, ivec2(xb * 4 + i, y)).xyz; + } + + // Decode Cb and Cr to shared memory, because neighboring blocks need it for interpolation. + cbcr[xb * 3 + 0] = indata[0].xz; + cbcr[xb * 3 + 1] = vec2(indata[1].y, indata[2].x); + cbcr[xb * 3 + 2] = vec2(indata[2].z, indata[3].y); + memoryBarrierShared(); + + float pix_y[6]; + pix_y[0] = indata[0].y; + pix_y[1] = indata[1].x; + pix_y[2] = indata[1].z; + pix_y[3] = indata[2].y; + pix_y[4] = indata[3].x; + pix_y[5] = indata[3].z; + + barrier(); + + // Interpolate the missing Cb/Cr pixels, taking care not to read past the end of the screen + // for pixels that we use for interpolation. + vec2 pix_cbcr[7]; + pix_cbcr[0] = indata[0].xz; + pix_cbcr[2] = cbcr[min(xb * 3 + 1, max_cbcr_x)]; + pix_cbcr[4] = cbcr[min(xb * 3 + 2, max_cbcr_x)]; + pix_cbcr[6] = cbcr[min(xb * 3 + 3, max_cbcr_x)]; + pix_cbcr[1] = 0.5 * (pix_cbcr[0] + pix_cbcr[2]); + pix_cbcr[3] = 0.5 * (pix_cbcr[2] + pix_cbcr[4]); + pix_cbcr[5] = 0.5 * (pix_cbcr[4] + pix_cbcr[6]); + + // Write the decoded pixels to the destination texture. + for (int i = 0; i < 6; ++i) { + vec4 outdata = vec4(pix_y[i], pix_cbcr[i].x, pix_cbcr[i].y, 1.0f); + imageStore(outbuf, ivec2(xb * 6 + i, y), outdata); + } +} +)"; + + Shader shader; + + GLuint shader_num = movit::compile_shader(shader_src, GL_COMPUTE_SHADER); + check_error(); + shader.glsl_program_num = glCreateProgram(); + check_error(); + glAttachShader(shader.glsl_program_num, shader_num); + check_error(); + glLinkProgram(shader.glsl_program_num); + check_error(); + + GLint success; + glGetProgramiv(shader.glsl_program_num, GL_LINK_STATUS, &success); + check_error(); + if (success == GL_FALSE) { + GLchar error_log[1024] = {0}; + glGetProgramInfoLog(shader.glsl_program_num, 1024, nullptr, error_log); + fprintf(stderr, "Error linking program: %s\n", error_log); + exit(1); + } + + shader.max_cbcr_x_pos = glGetUniformLocation(shader.glsl_program_num, "max_cbcr_x"); + check_error(); + shader.inbuf_pos = glGetUniformLocation(shader.glsl_program_num, "inbuf"); + check_error(); + shader.outbuf_pos = glGetUniformLocation(shader.glsl_program_num, "outbuf"); + check_error(); + + shaders.emplace(num_local_work_groups, shader); +} + +void v210Converter::convert(GLuint tex_src, GLuint tex_dst, unsigned width, unsigned height) +{ + precompile_shader(width); + unsigned num_local_work_groups = (width + 5) / 6; + const Shader &shader = shaders[num_local_work_groups]; + + glUseProgram(shader.glsl_program_num); + check_error(); + glUniform1i(shader.max_cbcr_x_pos, width / 2 - 1); + check_error(); + + // Bind the textures. + glUniform1i(shader.inbuf_pos, 0); + check_error(); + glUniform1i(shader.outbuf_pos, 1); + check_error(); + glBindImageTexture(0, tex_src, 0, GL_FALSE, 0, GL_READ_ONLY, GL_RGB10_A2); + check_error(); + glBindImageTexture(1, tex_dst, 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGB10_A2); + check_error(); + + // Actually run the shader. + glDispatchCompute(1, height, 1); + check_error(); + + glUseProgram(0); + check_error(); +} diff --git a/nageru/v210_converter.h b/nageru/v210_converter.h new file mode 100644 index 0000000..39c456f --- /dev/null +++ b/nageru/v210_converter.h @@ -0,0 +1,103 @@ +#ifndef _V210CONVERTER_H +#define _V210CONVERTER_H 1 + +// v210 is a 10-bit 4:2:2 interleaved Y'CbCr format, packing three values +// into a 32-bit int (leaving two unused bits at the top) with chroma being +// sub-sited with the left luma sample. Even though this 2:10:10:10-arrangement +// can be sampled from using the GL_RGB10_A2/GL_UNSIGNED_2_10_10_10_REV format, +// the placement of the Y', Cb and Cr parts within these ints is rather +// complicated, and thus hard to get a single Y'CbCr pixel from efficiently, +// especially on a GPU. Six pixels (six Y', three Cb, three Cr) are packed into +// four such ints in the following pattern (see e.g. the DeckLink documentation +// for reference): +// +// A B G R +// ----------------- +// X Cr0 Y0 Cb0 +// X Y2 Cb2 Y1 +// X Cb4 Y3 Cr2 +// X Y5 Cr4 Y4 +// +// This patterns repeats for as long as needed, with the additional constraint +// that stride must be divisible by 128 (or equivalently, 32 four-byte ints, +// or eight pixel groups representing 48 pixels in all). +// +// Thus, v210Converter allows you to convert from v210 to a more regular +// 4:4:4 format (upsampling Cb/Cr on the way, using linear interpolation) +// that the GPU supports natively, again in the form of GL_RGB10_A2 +// (with Y', Cb, Cr packed as R, G and B, respectively -- the “alpha” channel +// is always 1). +// +// It does this fairly efficiently using a compute shader, which means you'll +// need compute shader support (GL_ARB_compute_shader + GL_ARB_shader_image_load_store, +// or equivalently, OpenGL 4.3 or newer) to use it. There are many possible +// strategies for doing this in a compute shader, but I ended up settling +// a fairly simple one after some benchmarking; each work unit takes in +// a single four-int group and writes six samples, but as the interpolation +// needs the leftmost chroma samples from the work unit at the right, each line +// is put into a local work group. Cb/Cr is first decoded into shared memory +// (OpenGL guarantees at least 32 kB shared memory for the work group, which is +// enough for up to 6K video or so), and then the rest of the shuffling and +// writing happens. Each line can of course be converted entirely +// independently, so we can fire up as many such work groups as we have lines. +// +// On the Haswell GPU where I developed it (with single-channel memory), +// conversion takes about 1.4 ms for a 720p frame, so it should be possible to +// keep up multiple inputs at 720p60, although probably a faster machine is +// needed if we want to run e.g. heavy scaling filters in the same pipeline. +// (1.4 ms equates to about 35% of the theoretical memory bandwidth of +// 12.8 GB/sec, which is pretty good.) + +#include + +#include + +class v210Converter { +public: + ~v210Converter(); + + // Whether the current hardware and driver supports the compute shader + // necessary to do this conversion. + static bool has_hardware_support(); + + // Given an image width, returns the minimum number of 32-bit groups + // needed for each line. This can be used to size the input texture properly. + static GLuint get_minimum_v210_texture_width(unsigned width) + { + unsigned num_local_groups = (width + 5) / 6; + return 4 * num_local_groups; + } + + // Given an image width, returns the stride (in bytes) for each line. + static size_t get_v210_stride(unsigned width) + { + return (width + 47) / 48 * 128; + } + + // Since work groups need to be determined at shader compile time, + // each width needs potentially a different shader. You can call this + // function at startup to make sure a shader for the given width + // has been compiled, making sure you don't need to start an expensive + // compilation job while video is running if a new resolution comes along. + // This is not required, but generally recommended. + void precompile_shader(unsigned width); + + // Do the actual conversion. tex_src is assumed to be a GL_RGB10_A2 + // texture of at least [get_minimum_v210_texture_width(width), height]. + // tex_dst is assumed to be a GL_RGB10_A2 texture of exactly [width, height] + // (actually, other sizes will work fine, but be nonsensical). + // No textures will be allocated or deleted. + void convert(GLuint tex_src, GLuint tex_dst, unsigned width, unsigned height); + +private: + // Key is number of local groups, ie., ceil(width / 6). + struct Shader { + GLuint glsl_program_num = -1; + + // Uniform locations. + GLuint max_cbcr_x_pos = -1, inbuf_pos = -1, outbuf_pos = -1; + }; + std::map shaders; +}; + +#endif // !defined(_V210CONVERTER_H) diff --git a/nageru/video_encoder.cpp b/nageru/video_encoder.cpp new file mode 100644 index 0000000..6344b8c --- /dev/null +++ b/nageru/video_encoder.cpp @@ -0,0 +1,225 @@ +#include "video_encoder.h" + +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +#include "audio_encoder.h" +#include "defs.h" +#include "ffmpeg_raii.h" +#include "flags.h" +#include "httpd.h" +#include "mux.h" +#include "quicksync_encoder.h" +#include "timebase.h" +#include "x264_encoder.h" + +class RefCountedFrame; + +using namespace std; +using namespace movit; + +namespace { + +string generate_local_dump_filename(int frame) +{ + time_t now = time(NULL); + tm now_tm; + localtime_r(&now, &now_tm); + + char timestamp[64]; + strftime(timestamp, sizeof(timestamp), "%F-%T%z", &now_tm); + + // Use the frame number to disambiguate between two cuts starting + // on the same second. + char filename[256]; + snprintf(filename, sizeof(filename), "%s/%s%s-f%02d%s", + global_flags.recording_dir.c_str(), + LOCAL_DUMP_PREFIX, timestamp, frame % 100, LOCAL_DUMP_SUFFIX); + return filename; +} + +} // namespace + +VideoEncoder::VideoEncoder(ResourcePool *resource_pool, QSurface *surface, const std::string &va_display, int width, int height, HTTPD *httpd, DiskSpaceEstimator *disk_space_estimator) + : resource_pool(resource_pool), surface(surface), va_display(va_display), width(width), height(height), httpd(httpd), disk_space_estimator(disk_space_estimator) +{ + oformat = av_guess_format(global_flags.stream_mux_name.c_str(), nullptr, nullptr); + assert(oformat != nullptr); + if (global_flags.stream_audio_codec_name.empty()) { + stream_audio_encoder.reset(new AudioEncoder(AUDIO_OUTPUT_CODEC_NAME, DEFAULT_AUDIO_OUTPUT_BIT_RATE, oformat)); + } else { + stream_audio_encoder.reset(new AudioEncoder(global_flags.stream_audio_codec_name, global_flags.stream_audio_codec_bitrate, oformat)); + } + if (global_flags.x264_video_to_http || global_flags.x264_video_to_disk) { + x264_encoder.reset(new X264Encoder(oformat)); + } + + string filename = generate_local_dump_filename(/*frame=*/0); + quicksync_encoder.reset(new QuickSyncEncoder(filename, resource_pool, surface, va_display, width, height, oformat, x264_encoder.get(), disk_space_estimator)); + + open_output_stream(); + stream_audio_encoder->add_mux(stream_mux.get()); + quicksync_encoder->set_stream_mux(stream_mux.get()); + if (global_flags.x264_video_to_http) { + x264_encoder->add_mux(stream_mux.get()); + } +} + +VideoEncoder::~VideoEncoder() +{ + quicksync_encoder->shutdown(); + x264_encoder.reset(nullptr); + quicksync_encoder->close_file(); + quicksync_encoder.reset(nullptr); + while (quicksync_encoders_in_shutdown.load() > 0) { + usleep(10000); + } +} + +void VideoEncoder::do_cut(int frame) +{ + string filename = generate_local_dump_filename(frame); + printf("Starting new recording: %s\n", filename.c_str()); + + // Do the shutdown of the old encoder in a separate thread, since it can + // take some time (it needs to wait for all the frames in the queue to be + // done encoding, for one) and we are running on the main mixer thread. + // However, since this means both encoders could be sending packets at + // the same time, it means pts could come out of order to the stream mux, + // and we need to plug it until the shutdown is complete. + stream_mux->plug(); + lock(qs_mu, qs_audio_mu); + lock_guard lock1(qs_mu, adopt_lock), lock2(qs_audio_mu, adopt_lock); + QuickSyncEncoder *old_encoder = quicksync_encoder.release(); // When we go C++14, we can use move capture instead. + X264Encoder *old_x264_encoder = nullptr; + if (global_flags.x264_video_to_disk) { + old_x264_encoder = x264_encoder.release(); + } + thread([old_encoder, old_x264_encoder, this]{ + old_encoder->shutdown(); + delete old_x264_encoder; + old_encoder->close_file(); + stream_mux->unplug(); + + // We cannot delete the encoder here, as this thread has no OpenGL context. + // We'll deal with it in begin_frame(). + lock_guard lock(qs_mu); + qs_needing_cleanup.emplace_back(old_encoder); + }).detach(); + + if (global_flags.x264_video_to_disk) { + x264_encoder.reset(new X264Encoder(oformat)); + if (global_flags.x264_video_to_http) { + x264_encoder->add_mux(stream_mux.get()); + } + if (overriding_bitrate != 0) { + x264_encoder->change_bitrate(overriding_bitrate); + } + } + + quicksync_encoder.reset(new QuickSyncEncoder(filename, resource_pool, surface, va_display, width, height, oformat, x264_encoder.get(), disk_space_estimator)); + quicksync_encoder->set_stream_mux(stream_mux.get()); +} + +void VideoEncoder::change_x264_bitrate(unsigned rate_kbit) +{ + overriding_bitrate = rate_kbit; + x264_encoder->change_bitrate(rate_kbit); +} + +void VideoEncoder::add_audio(int64_t pts, std::vector audio) +{ + // Take only qs_audio_mu, since add_audio() is thread safe + // (we can only conflict with do_cut(), which takes qs_audio_mu) + // and we don't want to contend with begin_frame(). + { + lock_guard lock(qs_audio_mu); + quicksync_encoder->add_audio(pts, audio); + } + stream_audio_encoder->encode_audio(audio, pts + quicksync_encoder->global_delay()); +} + +bool VideoEncoder::is_zerocopy() const +{ + // Explicitly do _not_ take qs_mu; this is called from the mixer, + // and qs_mu might be contended. is_zerocopy() is thread safe + // and never called in parallel with do_cut() (both happen only + // from the mixer thread). + return quicksync_encoder->is_zerocopy(); +} + +bool VideoEncoder::begin_frame(int64_t pts, int64_t duration, movit::YCbCrLumaCoefficients ycbcr_coefficients, const std::vector &input_frames, GLuint *y_tex, GLuint *cbcr_tex) +{ + lock_guard lock(qs_mu); + qs_needing_cleanup.clear(); // Since we have an OpenGL context here, and are called regularly. + return quicksync_encoder->begin_frame(pts, duration, ycbcr_coefficients, input_frames, y_tex, cbcr_tex); +} + +RefCountedGLsync VideoEncoder::end_frame() +{ + lock_guard lock(qs_mu); + return quicksync_encoder->end_frame(); +} + +void VideoEncoder::open_output_stream() +{ + AVFormatContext *avctx = avformat_alloc_context(); + avctx->oformat = oformat; + + uint8_t *buf = (uint8_t *)av_malloc(MUX_BUFFER_SIZE); + avctx->pb = avio_alloc_context(buf, MUX_BUFFER_SIZE, 1, this, nullptr, nullptr, nullptr); + avctx->pb->write_data_type = &VideoEncoder::write_packet2_thunk; + avctx->pb->ignore_boundary_point = 1; + + Mux::Codec video_codec; + if (global_flags.uncompressed_video_to_http) { + video_codec = Mux::CODEC_NV12; + } else { + video_codec = Mux::CODEC_H264; + } + + avctx->flags = AVFMT_FLAG_CUSTOM_IO; + + string video_extradata; + if (global_flags.x264_video_to_http || global_flags.x264_video_to_disk) { + video_extradata = x264_encoder->get_global_headers(); + } + + stream_mux.reset(new Mux(avctx, width, height, video_codec, video_extradata, stream_audio_encoder->get_codec_parameters().get(), COARSE_TIMEBASE, + /*write_callback=*/nullptr, Mux::WRITE_FOREGROUND, { &stream_mux_metrics })); + stream_mux_metrics.init({{ "destination", "http" }}); +} + +int VideoEncoder::write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time) +{ + VideoEncoder *video_encoder = (VideoEncoder *)opaque; + return video_encoder->write_packet2(buf, buf_size, type, time); +} + +int VideoEncoder::write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time) +{ + if (type == AVIO_DATA_MARKER_SYNC_POINT || type == AVIO_DATA_MARKER_BOUNDARY_POINT) { + seen_sync_markers = true; + } else if (type == AVIO_DATA_MARKER_UNKNOWN && !seen_sync_markers) { + // We don't know if this is a keyframe or not (the muxer could + // avoid marking it), so we just have to make the best of it. + type = AVIO_DATA_MARKER_SYNC_POINT; + } + + if (type == AVIO_DATA_MARKER_HEADER) { + stream_mux_header.append((char *)buf, buf_size); + httpd->set_header(stream_mux_header); + } else { + httpd->add_data((char *)buf, buf_size, type == AVIO_DATA_MARKER_SYNC_POINT, time, AVRational{ AV_TIME_BASE, 1 }); + } + return buf_size; +} + diff --git a/nageru/video_encoder.h b/nageru/video_encoder.h new file mode 100644 index 0000000..21595a3 --- /dev/null +++ b/nageru/video_encoder.h @@ -0,0 +1,108 @@ +// A class to orchestrate the concept of video encoding. Will keep track of +// the muxes to stream and disk, the QuickSyncEncoder, and also the X264Encoder +// (for the stream) if there is one. + +#ifndef _VIDEO_ENCODER_H +#define _VIDEO_ENCODER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +} + +#include "mux.h" +#include "ref_counted_gl_sync.h" + +class AudioEncoder; +class DiskSpaceEstimator; +class HTTPD; +class Mux; +class QSurface; +class QuickSyncEncoder; +class RefCountedFrame; +class X264Encoder; + +namespace movit { +class ResourcePool; +} // namespace movit + +class VideoEncoder { +public: + VideoEncoder(movit::ResourcePool *resource_pool, QSurface *surface, const std::string &va_display, int width, int height, HTTPD *httpd, DiskSpaceEstimator *disk_space_estimator); + ~VideoEncoder(); + + void add_audio(int64_t pts, std::vector audio); + + bool is_zerocopy() const; + + // Allocate a frame to render into. The returned two textures + // are yours to render into (build them into an FBO). + // Call end_frame() when you're done. + // + // The semantics of y_tex and cbcr_tex depend on is_zerocopy(): + // + // - If false, the are input parameters, ie., the caller + // allocates textures. (The contents are not read before + // end_frame() is called.) + // - If true, they are output parameters, ie., VideoEncoder + // allocates textures and borrow them to you for rendering. + // In this case, after end_frame(), you are no longer allowed + // to use the textures; they are torn down and given to the + // H.264 encoder. + bool begin_frame(int64_t pts, int64_t duration, movit::YCbCrLumaCoefficients ycbcr_coefficients, const std::vector &input_frames, GLuint *y_tex, GLuint *cbcr_tex); + + // Call after you are done rendering into the frame; at this point, + // y_tex and cbcr_tex will be assumed done, and handed over to the + // encoder. The returned fence is purely a convenience; you do not + // need to use it for anything, but it's useful if you wanted to set + // one anyway. + RefCountedGLsync end_frame(); + + // Does a cut of the disk stream immediately ("frame" is used for the filename only). + void do_cut(int frame); + + void change_x264_bitrate(unsigned rate_kbit); + +private: + void open_output_stream(); + static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time); + int write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time); + + AVOutputFormat *oformat; + mutable std::mutex qs_mu, qs_audio_mu; + std::unique_ptr quicksync_encoder; // Under _and_ . + movit::ResourcePool *resource_pool; + QSurface *surface; + std::string va_display; + int width, height; + HTTPD *httpd; + DiskSpaceEstimator *disk_space_estimator; + + bool seen_sync_markers = false; + + std::unique_ptr stream_mux; // To HTTP. + std::unique_ptr stream_audio_encoder; + std::unique_ptr x264_encoder; // nullptr if not using x264. + + std::string stream_mux_header; + MuxMetrics stream_mux_metrics; + + std::atomic quicksync_encoders_in_shutdown{0}; + std::atomic overriding_bitrate{0}; + + // Encoders that are shutdown, but need to call release_gl_resources() + // (or be deleted) from some thread with an OpenGL context. + std::vector> qs_needing_cleanup; // Under . +}; + +#endif diff --git a/nageru/vu_common.cpp b/nageru/vu_common.cpp new file mode 100644 index 0000000..171f50d --- /dev/null +++ b/nageru/vu_common.cpp @@ -0,0 +1,73 @@ +#include "vu_common.h" + +#include +#include +#include +#include + +using namespace std; + +double lufs_to_pos(float level_lu, int height, float min_level, float max_level) +{ + // Note: “max” is the loudest level, but y=0 is top of screen. + + // Handle -inf. + if (level_lu < min_level) { + return height - 1; + } + + double y = height * (level_lu - max_level) / (min_level - max_level); + y = max(y, 0); + y = min(y, height - 1); + + // If we are big enough, snap to pixel grid instead of antialiasing + // the edges; the unevenness will be less noticeable than the blurriness. + double height_per_level = height / (max_level - min_level) - 2.0; + if (height_per_level >= 10.0) { + y = round(y); + } + + return y; +} + +void draw_vu_meter(QPainter &painter, int width, int height, int horizontal_margin, double segment_margin, bool is_on, float min_level, float max_level, bool flip, int y_offset) +{ + painter.fillRect(horizontal_margin, y_offset, width - 2 * horizontal_margin, height, Qt::black); + + for (int y = 0; y < height; ++y) { + // Find coverage of “on” rectangles in this pixel row. + double coverage = 0.0; + for (int level = floor(min_level); level <= ceil(max_level); ++level) { + double min_y = lufs_to_pos(level + 1.0, height, min_level, max_level) + segment_margin * 0.5; + double max_y = lufs_to_pos(level, height, min_level, max_level) - segment_margin * 0.5; + min_y = std::max(min_y, y); + min_y = std::min(min_y, y + 1); + max_y = std::max(max_y, y); + max_y = std::min(max_y, y + 1); + coverage += max_y - min_y; + } + + double on_r, on_g, on_b; + if (is_on) { + double t = double(y) / height; + if (t <= 0.5) { + on_r = 1.0; + on_g = 2.0 * t; + on_b = 0.0; + } else { + on_r = 1.0 - 2.0 * (t - 0.5); + on_g = 1.0; + on_b = 0.0; + } + } else { + on_r = on_g = on_b = 0.05; + } + + // Correct for coverage and do a simple gamma correction. + int r = lrintf(255 * pow(on_r * coverage, 1.0 / 2.2)); + int g = lrintf(255 * pow(on_g * coverage, 1.0 / 2.2)); + int b = lrintf(255 * pow(on_b * coverage, 1.0 / 2.2)); + int draw_y = flip ? (height - y - 1) : y; + painter.fillRect(horizontal_margin, draw_y + y_offset, width - 2 * horizontal_margin, 1, QColor(r, g, b)); + } +} diff --git a/nageru/vu_common.h b/nageru/vu_common.h new file mode 100644 index 0000000..602de8b --- /dev/null +++ b/nageru/vu_common.h @@ -0,0 +1,10 @@ +#ifndef _VU_COMMON_H +#define _VU_COMMON_H 1 + +class QPainter; + +double lufs_to_pos(float level_lu, int height, float min_level, float max_level); + +void draw_vu_meter(QPainter &painter, int width, int height, int horizontal_margin, double segment_margin, bool is_on, float min_level, float max_level, bool flip, int y_offset = 0); + +#endif // !defined(_VU_COMMON_H) diff --git a/nageru/vumeter.cpp b/nageru/vumeter.cpp new file mode 100644 index 0000000..b697a83 --- /dev/null +++ b/nageru/vumeter.cpp @@ -0,0 +1,76 @@ +#include "vumeter.h" + +#include +#include +#include "vu_common.h" + +class QPaintEvent; +class QResizeEvent; + +using namespace std; + +VUMeter::VUMeter(QWidget *parent) + : QWidget(parent) +{ +} + +void VUMeter::resizeEvent(QResizeEvent *event) +{ + recalculate_pixmaps(); +} + +void VUMeter::paintEvent(QPaintEvent *event) +{ + QPainter painter(this); + + float level_lufs[2], peak_lufs[2]; + { + unique_lock lock(level_mutex); + level_lufs[0] = this->level_lufs[0]; + level_lufs[1] = this->level_lufs[1]; + peak_lufs[0] = this->peak_lufs[0]; + peak_lufs[1] = this->peak_lufs[1]; + } + + int mid = width() / 2; + + for (unsigned channel = 0; channel < 2; ++channel) { + int left = (channel == 0) ? 0 : mid; + int right = (channel == 0) ? mid : width(); + float level_lu = level_lufs[channel] - ref_level_lufs; + int on_pos = lrint(lufs_to_pos(level_lu, height())); + + QRect off_rect(left, 0, right - left, on_pos); + QRect on_rect(left, on_pos, right - left, height() - on_pos); + + painter.drawPixmap(off_rect, off_pixmap, off_rect); + painter.drawPixmap(on_rect, on_pixmap, on_rect); + + float peak_lu = peak_lufs[channel] - ref_level_lufs; + if (peak_lu >= min_level && peak_lu <= max_level) { + int peak_pos = lrint(lufs_to_pos(peak_lu, height())); + QRect peak_rect(left, peak_pos - 1, right, 2); + painter.drawPixmap(peak_rect, full_on_pixmap, peak_rect); + } + } +} + +void VUMeter::recalculate_pixmaps() +{ + full_on_pixmap = QPixmap(width(), height()); + QPainter full_on_painter(&full_on_pixmap); + full_on_painter.fillRect(0, 0, width(), height(), parentWidget()->palette().window()); + draw_vu_meter(full_on_painter, width(), height(), 0, 0.0, true, min_level, max_level, /*flip=*/false); + + float margin = 0.5 * (width() - 20); + + on_pixmap = QPixmap(width(), height()); + QPainter on_painter(&on_pixmap); + on_painter.fillRect(0, 0, width(), height(), parentWidget()->palette().window()); + draw_vu_meter(on_painter, width(), height(), margin, 2.0, true, min_level, max_level, /*flip=*/false); + + off_pixmap = QPixmap(width(), height()); + QPainter off_painter(&off_pixmap); + off_painter.fillRect(0, 0, width(), height(), parentWidget()->palette().window()); + draw_vu_meter(off_painter, width(), height(), margin, 2.0, false, min_level, max_level, /*flip=*/false); +} diff --git a/nageru/vumeter.h b/nageru/vumeter.h new file mode 100644 index 0000000..7a94200 --- /dev/null +++ b/nageru/vumeter.h @@ -0,0 +1,80 @@ +#ifndef VUMETER_H +#define VUMETER_H + +#include +#include +#include +#include +#include + +#include "vu_common.h" + +class QObject; +class QPaintEvent; +class QResizeEvent; + +class VUMeter : public QWidget +{ + Q_OBJECT + +public: + VUMeter(QWidget *parent); + + void set_level(float level_lufs) { + set_level(level_lufs, level_lufs); + } + + void set_level(float level_lufs_left, float level_lufs_right) { + std::unique_lock lock(level_mutex); + this->level_lufs[0] = level_lufs_left; + this->level_lufs[1] = level_lufs_right; + QMetaObject::invokeMethod(this, "update", Qt::AutoConnection); + } + + void set_peak(float peak_lufs) { + set_peak(peak_lufs, peak_lufs); + } + + void set_peak(float peak_lufs_left, float peak_lufs_right) { + std::unique_lock lock(level_mutex); + this->peak_lufs[0] = peak_lufs_left; + this->peak_lufs[1] = peak_lufs_right; + QMetaObject::invokeMethod(this, "update", Qt::AutoConnection); + } + + double lufs_to_pos(float level_lu, int height) + { + return ::lufs_to_pos(level_lu, height, min_level, max_level); + } + + void set_min_level(float min_level) + { + this->min_level = min_level; + recalculate_pixmaps(); + } + + void set_max_level(float max_level) + { + this->max_level = max_level; + recalculate_pixmaps(); + } + + void set_ref_level(float ref_level_lufs) + { + this->ref_level_lufs = ref_level_lufs; + } + +private: + void resizeEvent(QResizeEvent *event) override; + void paintEvent(QPaintEvent *event) override; + void recalculate_pixmaps(); + + std::mutex level_mutex; + float level_lufs[2] { -HUGE_VALF, -HUGE_VALF }; + float peak_lufs[2] { -HUGE_VALF, -HUGE_VALF }; + float min_level = -18.0f, max_level = 9.0f, ref_level_lufs = -23.0f; + + QPixmap full_on_pixmap, on_pixmap, off_pixmap; +}; + +#endif diff --git a/nageru/x264_dynamic.cpp b/nageru/x264_dynamic.cpp new file mode 100644 index 0000000..f8b63ce --- /dev/null +++ b/nageru/x264_dynamic.cpp @@ -0,0 +1,92 @@ +#include "x264_dynamic.h" + +#include +#include +#include +#include +#include +#include + +#include + +using namespace std; + +X264Dynamic load_x264_for_bit_depth(unsigned depth) +{ + X264Dynamic dyn; +#if defined(X264_BIT_DEPTH) && X264_BIT_DEPTH == 0 + bool suitable = true; // x264 compiled to support all bit depths. +#elif defined(X264_BIT_DEPTH) + bool suitable = X264_BIT_DEPTH >= depth; +#else + bool suitable = unsigned(x264_bit_depth) >= depth; +#endif + if (suitable) { + // Just use the one we are linked to. + dyn.handle = nullptr; + dyn.x264_encoder_close = x264_encoder_close; + dyn.x264_encoder_delayed_frames = x264_encoder_delayed_frames; + dyn.x264_encoder_encode = x264_encoder_encode; + dyn.x264_encoder_headers = x264_encoder_headers; + dyn.x264_encoder_open = x264_encoder_open; + dyn.x264_encoder_parameters = x264_encoder_parameters; + dyn.x264_encoder_reconfig = x264_encoder_reconfig; + dyn.x264_param_apply_profile = x264_param_apply_profile; + dyn.x264_param_default_preset = x264_param_default_preset; + dyn.x264_param_parse = x264_param_parse; + dyn.x264_picture_init = x264_picture_init; + return dyn; + } + + // Uh-oh, our currently loaded library doesn't have the required support. + // Let's try to dynamically load a 10-bit version; in particular, Debian + // has a version in /usr/lib/x86_64-linux-gnu/x264-10bit/libx264.so., + // so we try to figure out where our libx264 comes from, and modify that path. + string x264_dir, x264_suffix; + void *handle = dlopen(nullptr, RTLD_NOW); + link_map *m; + int err = dlinfo(handle, RTLD_DI_LINKMAP, &m); + assert(err != -1); + for ( ; m != nullptr; m = m->l_next) { + if (m->l_name == nullptr) { + continue; + } + const char *ptr = strstr(m->l_name, "/libx264.so."); + if (ptr != nullptr) { + x264_dir = string(m->l_name, ptr - m->l_name); + x264_suffix = string(ptr, (m->l_name + strlen(m->l_name)) - ptr); + break; + } + } + dlclose(handle); + + if (x264_dir.empty()) { + fprintf(stderr, "ERROR: Requested %d-bit x264, but is not linked to such an x264, and could not find one.\n", + depth); + exit(1); + } + + string x264_10b_string = x264_dir + "/x264-10bit" + x264_suffix; + void *x264_dlhandle = dlopen(x264_10b_string.c_str(), RTLD_NOW); + if (x264_dlhandle == nullptr) { + fprintf(stderr, "ERROR: Requested %d-bit x264, but is not linked to such an x264, and %s would not load.\n", + depth, x264_10b_string.c_str()); + exit(1); + } + + dyn.handle = x264_dlhandle; + dyn.x264_encoder_close = (decltype(::x264_encoder_close) *)dlsym(x264_dlhandle, "x264_encoder_close"); + dyn.x264_encoder_delayed_frames = (decltype(::x264_encoder_delayed_frames) *)dlsym(x264_dlhandle, "x264_encoder_delayed_frames"); + dyn.x264_encoder_encode = (decltype(::x264_encoder_encode) *)dlsym(x264_dlhandle, "x264_encoder_encode"); + dyn.x264_encoder_headers = (decltype(::x264_encoder_headers) *)dlsym(x264_dlhandle, "x264_encoder_headers"); + char x264_encoder_open_symname[256]; + snprintf(x264_encoder_open_symname, sizeof(x264_encoder_open_symname), "x264_encoder_open_%d", X264_BUILD); + dyn.x264_encoder_open = (decltype(::x264_encoder_open) *)dlsym(x264_dlhandle, x264_encoder_open_symname); + dyn.x264_encoder_parameters = (decltype(::x264_encoder_parameters) *)dlsym(x264_dlhandle, "x264_encoder_parameters"); + dyn.x264_encoder_reconfig = (decltype(::x264_encoder_reconfig) *)dlsym(x264_dlhandle, "x264_encoder_reconfig"); + dyn.x264_param_apply_profile = (decltype(::x264_param_apply_profile) *)dlsym(x264_dlhandle, "x264_param_apply_profile"); + dyn.x264_param_default_preset = (decltype(::x264_param_default_preset) *)dlsym(x264_dlhandle, "x264_param_default_preset"); + dyn.x264_param_parse = (decltype(::x264_param_parse) *)dlsym(x264_dlhandle, "x264_param_parse"); + dyn.x264_picture_init = (decltype(::x264_picture_init) *)dlsym(x264_dlhandle, "x264_picture_init"); + return dyn; +} diff --git a/nageru/x264_dynamic.h b/nageru/x264_dynamic.h new file mode 100644 index 0000000..27e5202 --- /dev/null +++ b/nageru/x264_dynamic.h @@ -0,0 +1,28 @@ +#ifndef _X264_DYNAMIC_H +#define _X264_DYNAMIC_H 1 + +// A helper to load 10-bit x264 if needed. + +#include + +extern "C" { +#include +} + +struct X264Dynamic { + void *handle; // If not nullptr, needs to be dlclose()d. + decltype(::x264_encoder_close) *x264_encoder_close; + decltype(::x264_encoder_delayed_frames) *x264_encoder_delayed_frames; + decltype(::x264_encoder_encode) *x264_encoder_encode; + decltype(::x264_encoder_headers) *x264_encoder_headers; + decltype(::x264_encoder_open) *x264_encoder_open; + decltype(::x264_encoder_parameters) *x264_encoder_parameters; + decltype(::x264_encoder_reconfig) *x264_encoder_reconfig; + decltype(::x264_param_apply_profile) *x264_param_apply_profile; + decltype(::x264_param_default_preset) *x264_param_default_preset; + decltype(::x264_param_parse) *x264_param_parse; + decltype(::x264_picture_init) *x264_picture_init; +}; +X264Dynamic load_x264_for_bit_depth(unsigned depth); + +#endif // !defined(_X264_DYNAMIC_H) diff --git a/nageru/x264_encoder.cpp b/nageru/x264_encoder.cpp new file mode 100644 index 0000000..8463d1b --- /dev/null +++ b/nageru/x264_encoder.cpp @@ -0,0 +1,436 @@ +#include "x264_encoder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "defs.h" +#include "flags.h" +#include "metrics.h" +#include "mux.h" +#include "print_latency.h" +#include "timebase.h" +#include "x264_dynamic.h" +#include "x264_speed_control.h" + +extern "C" { +#include +#include +} + +using namespace movit; +using namespace std; +using namespace std::chrono; +using namespace std::placeholders; + +namespace { + +// X264Encoder can be restarted if --record-x264-video is set, so make these +// metrics global. +atomic metric_x264_queued_frames{0}; +atomic metric_x264_max_queued_frames{X264_QUEUE_LENGTH}; +atomic metric_x264_dropped_frames{0}; +atomic metric_x264_output_frames_i{0}; +atomic metric_x264_output_frames_p{0}; +atomic metric_x264_output_frames_b{0}; +Histogram metric_x264_crf; +LatencyHistogram x264_latency_histogram; +once_flag x264_metrics_inited; + +void update_vbv_settings(x264_param_t *param) +{ + if (global_flags.x264_bitrate == -1) { + return; + } + if (global_flags.x264_vbv_buffer_size < 0) { + param->rc.i_vbv_buffer_size = param->rc.i_bitrate; // One-second VBV. + } else { + param->rc.i_vbv_buffer_size = global_flags.x264_vbv_buffer_size; + } + if (global_flags.x264_vbv_max_bitrate < 0) { + param->rc.i_vbv_max_bitrate = param->rc.i_bitrate; // CBR. + } else { + param->rc.i_vbv_max_bitrate = global_flags.x264_vbv_max_bitrate; + } +} + +} // namespace + +X264Encoder::X264Encoder(AVOutputFormat *oformat) + : wants_global_headers(oformat->flags & AVFMT_GLOBALHEADER), + dyn(load_x264_for_bit_depth(global_flags.x264_bit_depth)) +{ + call_once(x264_metrics_inited, [](){ + global_metrics.add("x264_queued_frames", &metric_x264_queued_frames, Metrics::TYPE_GAUGE); + global_metrics.add("x264_max_queued_frames", &metric_x264_max_queued_frames, Metrics::TYPE_GAUGE); + global_metrics.add("x264_dropped_frames", &metric_x264_dropped_frames); + global_metrics.add("x264_output_frames", {{ "type", "i" }}, &metric_x264_output_frames_i); + global_metrics.add("x264_output_frames", {{ "type", "p" }}, &metric_x264_output_frames_p); + global_metrics.add("x264_output_frames", {{ "type", "b" }}, &metric_x264_output_frames_b); + + metric_x264_crf.init_uniform(50); + global_metrics.add("x264_crf", &metric_x264_crf); + x264_latency_histogram.init("x264"); + }); + + size_t bytes_per_pixel = global_flags.x264_bit_depth > 8 ? 2 : 1; + frame_pool.reset(new uint8_t[global_flags.width * global_flags.height * 2 * bytes_per_pixel * X264_QUEUE_LENGTH]); + for (unsigned i = 0; i < X264_QUEUE_LENGTH; ++i) { + free_frames.push(frame_pool.get() + i * (global_flags.width * global_flags.height * 2 * bytes_per_pixel)); + } + encoder_thread = thread(&X264Encoder::encoder_thread_func, this); +} + +X264Encoder::~X264Encoder() +{ + should_quit = true; + queued_frames_nonempty.notify_all(); + encoder_thread.join(); + if (dyn.handle) { + dlclose(dyn.handle); + } +} + +void X264Encoder::add_frame(int64_t pts, int64_t duration, YCbCrLumaCoefficients ycbcr_coefficients, const uint8_t *data, const ReceivedTimestamps &received_ts) +{ + assert(!should_quit); + + QueuedFrame qf; + qf.pts = pts; + qf.duration = duration; + qf.ycbcr_coefficients = ycbcr_coefficients; + qf.received_ts = received_ts; + + { + lock_guard lock(mu); + if (free_frames.empty()) { + fprintf(stderr, "WARNING: x264 queue full, dropping frame with pts %ld\n", pts); + ++metric_x264_dropped_frames; + return; + } + + qf.data = free_frames.front(); + free_frames.pop(); + } + + size_t bytes_per_pixel = global_flags.x264_bit_depth > 8 ? 2 : 1; + memcpy(qf.data, data, global_flags.width * global_flags.height * 2 * bytes_per_pixel); + + { + lock_guard lock(mu); + queued_frames.push(qf); + queued_frames_nonempty.notify_all(); + metric_x264_queued_frames = queued_frames.size(); + } +} + +void X264Encoder::init_x264() +{ + x264_param_t param; + dyn.x264_param_default_preset(¶m, global_flags.x264_preset.c_str(), global_flags.x264_tune.c_str()); + + param.i_width = global_flags.width; + param.i_height = global_flags.height; + param.i_csp = X264_CSP_NV12; + if (global_flags.x264_bit_depth > 8) { + param.i_csp |= X264_CSP_HIGH_DEPTH; + } + param.b_vfr_input = 1; + param.i_timebase_num = 1; + param.i_timebase_den = TIMEBASE; + param.i_keyint_max = 50; // About one second. + if (global_flags.x264_speedcontrol) { + param.i_frame_reference = 16; // Because speedcontrol is never allowed to change this above what we set at start. + } +#if X264_BUILD >= 153 + param.i_bitdepth = global_flags.x264_bit_depth; +#endif + + // NOTE: These should be in sync with the ones in quicksync_encoder.cpp (sps_rbsp()). + param.vui.i_vidformat = 5; // Unspecified. + param.vui.b_fullrange = 0; + param.vui.i_colorprim = 1; // BT.709. + param.vui.i_transfer = 13; // sRGB. + if (global_flags.ycbcr_rec709_coefficients) { + param.vui.i_colmatrix = 1; // BT.709. + } else { + param.vui.i_colmatrix = 6; // BT.601/SMPTE 170M. + } + + if (!isinf(global_flags.x264_crf)) { + param.rc.i_rc_method = X264_RC_CRF; + param.rc.f_rf_constant = global_flags.x264_crf; + } else { + param.rc.i_rc_method = X264_RC_ABR; + param.rc.i_bitrate = global_flags.x264_bitrate; + } + update_vbv_settings(¶m); + if (param.rc.i_vbv_max_bitrate > 0) { + // If the user wants VBV control to cap the max rate, it is + // also reasonable to assume that they are fine with the stream + // constantly being around that rate even for very low-complexity + // content; the obvious and extreme example being a static + // black picture. + // + // One would think it's fine to have low-complexity content use + // less bitrate, but it seems to cause problems in practice; + // e.g. VLC seems to often drop the stream (similar to a buffer + // underrun) in such cases, but only when streaming from Nageru, + // not when reading a dump of the same stream from disk. + // I'm not 100% sure whether it's in VLC (possibly some buffering + // in the HTTP layer), in microhttpd or somewhere in Nageru itself, + // but it's a typical case of problems that can arise. Similarly, + // TCP's congestion control is not always fond of the rate staying + // low for a while and then rising quickly -- a variation on the same + // problem. + // + // We solve this by simply asking x264 to fill in dummy bits + // in these cases, so that the bitrate stays reasonable constant. + // It's a waste of bandwidth, but it makes things go much more + // smoothly in these cases. (We don't do it if VBV control is off + // in general, not the least because it makes no sense and x264 + // thus ignores the parameter.) + param.rc.b_filler = 1; + } + + // Occasionally players have problem with extremely low quantizers; + // be on the safe side. Shouldn't affect quality in any meaningful way. + param.rc.i_qp_min = 5; + + for (const string &str : global_flags.x264_extra_param) { + const size_t pos = str.find(','); + if (pos == string::npos) { + if (dyn.x264_param_parse(¶m, str.c_str(), nullptr) != 0) { + fprintf(stderr, "ERROR: x264 rejected parameter '%s'\n", str.c_str()); + } + } else { + const string key = str.substr(0, pos); + const string value = str.substr(pos + 1); + if (dyn.x264_param_parse(¶m, key.c_str(), value.c_str()) != 0) { + fprintf(stderr, "ERROR: x264 rejected parameter '%s' set to '%s'\n", + key.c_str(), value.c_str()); + } + } + } + + if (global_flags.x264_bit_depth > 8) { + dyn.x264_param_apply_profile(¶m, "high10"); + } else { + dyn.x264_param_apply_profile(¶m, "high"); + } + + param.b_repeat_headers = !wants_global_headers; + + x264 = dyn.x264_encoder_open(¶m); + if (x264 == nullptr) { + fprintf(stderr, "ERROR: x264 initialization failed.\n"); + exit(1); + } + + if (global_flags.x264_speedcontrol) { + speed_control.reset(new X264SpeedControl(x264, /*f_speed=*/1.0f, X264_QUEUE_LENGTH, /*f_buffer_init=*/1.0f)); + } + + if (wants_global_headers) { + x264_nal_t *nal; + int num_nal; + + dyn.x264_encoder_headers(x264, &nal, &num_nal); + + for (int i = 0; i < num_nal; ++i) { + if (nal[i].i_type == NAL_SEI) { + // Don't put the SEI in extradata; make it part of the first frame instead. + buffered_sei += string((const char *)nal[i].p_payload, nal[i].i_payload); + } else { + global_headers += string((const char *)nal[i].p_payload, nal[i].i_payload); + } + } + } +} + +void X264Encoder::encoder_thread_func() +{ + if (nice(5) == -1) { // Note that x264 further nices some of its threads. + perror("nice()"); + // No exit; it's not fatal. + } + pthread_setname_np(pthread_self(), "x264_encode"); + init_x264(); + x264_init_done = true; + + bool frames_left; + + do { + QueuedFrame qf; + + // Wait for a queued frame, then dequeue it. + { + unique_lock lock(mu); + queued_frames_nonempty.wait(lock, [this]() { return !queued_frames.empty() || should_quit; }); + if (!queued_frames.empty()) { + qf = queued_frames.front(); + queued_frames.pop(); + } else { + qf.pts = -1; + qf.duration = -1; + qf.data = nullptr; + } + + metric_x264_queued_frames = queued_frames.size(); + frames_left = !queued_frames.empty(); + } + + encode_frame(qf); + + { + lock_guard lock(mu); + free_frames.push(qf.data); + } + + // We should quit only if the should_quit flag is set _and_ we have nothing + // in either queue. + } while (!should_quit || frames_left || dyn.x264_encoder_delayed_frames(x264) > 0); + + dyn.x264_encoder_close(x264); +} + +void X264Encoder::encode_frame(X264Encoder::QueuedFrame qf) +{ + x264_nal_t *nal = nullptr; + int num_nal = 0; + x264_picture_t pic; + x264_picture_t *input_pic = nullptr; + + if (qf.data) { + dyn.x264_picture_init(&pic); + + pic.i_pts = qf.pts; + if (global_flags.x264_bit_depth > 8) { + pic.img.i_csp = X264_CSP_NV12 | X264_CSP_HIGH_DEPTH; + pic.img.i_plane = 2; + pic.img.plane[0] = qf.data; + pic.img.i_stride[0] = global_flags.width * sizeof(uint16_t); + pic.img.plane[1] = qf.data + global_flags.width * global_flags.height * sizeof(uint16_t); + pic.img.i_stride[1] = global_flags.width / 2 * sizeof(uint32_t); + } else { + pic.img.i_csp = X264_CSP_NV12; + pic.img.i_plane = 2; + pic.img.plane[0] = qf.data; + pic.img.i_stride[0] = global_flags.width; + pic.img.plane[1] = qf.data + global_flags.width * global_flags.height; + pic.img.i_stride[1] = global_flags.width / 2 * sizeof(uint16_t); + } + pic.opaque = reinterpret_cast(intptr_t(qf.duration)); + + input_pic = &pic; + + frames_being_encoded[qf.pts] = qf.received_ts; + } + + unsigned new_rate = new_bitrate_kbit.load(); // Can be 0 for no change. + if (speed_control) { + speed_control->set_config_override_function(bind(&speed_control_override_func, new_rate, qf.ycbcr_coefficients, _1)); + } else { + x264_param_t param; + dyn.x264_encoder_parameters(x264, ¶m); + speed_control_override_func(new_rate, qf.ycbcr_coefficients, ¶m); + dyn.x264_encoder_reconfig(x264, ¶m); + } + + if (speed_control) { + float queue_fill_ratio; + { + lock_guard lock(mu); + queue_fill_ratio = float(free_frames.size()) / X264_QUEUE_LENGTH; + } + speed_control->before_frame(queue_fill_ratio, X264_QUEUE_LENGTH, 1e6 * qf.duration / TIMEBASE); + } + dyn.x264_encoder_encode(x264, &nal, &num_nal, input_pic, &pic); + if (speed_control) { + speed_control->after_frame(); + } + + if (num_nal == 0) return; + + if (IS_X264_TYPE_I(pic.i_type)) { + ++metric_x264_output_frames_i; + } else if (IS_X264_TYPE_B(pic.i_type)) { + ++metric_x264_output_frames_b; + } else { + ++metric_x264_output_frames_p; + } + + metric_x264_crf.count_event(pic.prop.f_crf_avg); + + if (frames_being_encoded.count(pic.i_pts)) { + ReceivedTimestamps received_ts = frames_being_encoded[pic.i_pts]; + frames_being_encoded.erase(pic.i_pts); + + static int frameno = 0; + print_latency("Current x264 latency (video inputs → network mux):", + received_ts, (pic.i_type == X264_TYPE_B || pic.i_type == X264_TYPE_BREF), + &frameno, &x264_latency_histogram); + } else { + assert(false); + } + + // We really need one AVPacket for the entire frame, it seems, + // so combine it all. + size_t num_bytes = buffered_sei.size(); + for (int i = 0; i < num_nal; ++i) { + num_bytes += nal[i].i_payload; + } + + unique_ptr data(new uint8_t[num_bytes]); + uint8_t *ptr = data.get(); + + if (!buffered_sei.empty()) { + memcpy(ptr, buffered_sei.data(), buffered_sei.size()); + ptr += buffered_sei.size(); + buffered_sei.clear(); + } + for (int i = 0; i < num_nal; ++i) { + memcpy(ptr, nal[i].p_payload, nal[i].i_payload); + ptr += nal[i].i_payload; + } + + AVPacket pkt; + memset(&pkt, 0, sizeof(pkt)); + pkt.buf = nullptr; + pkt.data = data.get(); + pkt.size = num_bytes; + pkt.stream_index = 0; + if (pic.b_keyframe) { + pkt.flags = AV_PKT_FLAG_KEY; + } else { + pkt.flags = 0; + } + pkt.duration = reinterpret_cast(pic.opaque); + + for (Mux *mux : muxes) { + mux->add_packet(pkt, pic.i_pts, pic.i_dts); + } +} + +void X264Encoder::speed_control_override_func(unsigned bitrate_kbit, movit::YCbCrLumaCoefficients ycbcr_coefficients, x264_param_t *param) +{ + if (bitrate_kbit != 0) { + param->rc.i_bitrate = bitrate_kbit; + update_vbv_settings(param); + } + + if (ycbcr_coefficients == YCBCR_REC_709) { + param->vui.i_colmatrix = 1; // BT.709. + } else { + assert(ycbcr_coefficients == YCBCR_REC_601); + param->vui.i_colmatrix = 6; // BT.601/SMPTE 170M. + } +} diff --git a/nageru/x264_encoder.h b/nageru/x264_encoder.h new file mode 100644 index 0000000..687bf71 --- /dev/null +++ b/nageru/x264_encoder.h @@ -0,0 +1,121 @@ +// A wrapper around x264, to encode video in higher quality than Quick Sync +// can give us. We maintain a queue of uncompressed Y'CbCr frames (of 50 frames, +// so a little under 100 MB at 720p), then have a separate thread pull out +// those threads as fast as we can to give it to x264 for encoding. +// +// The encoding threads are niced down because mixing is more important than +// encoding; if we lose frames in mixing, we'll lose frames to disk _and_ +// to the stream, as where if we lose frames in encoding, we'll lose frames +// to the stream only, so the latter is strictly better. More importantly, +// this allows speedcontrol to do its thing without disturbing the mixer. + +#ifndef _X264ENCODE_H +#define _X264ENCODE_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +#include + +#include "defs.h" +#include "metrics.h" +#include "print_latency.h" +#include "x264_dynamic.h" + +class Mux; +class X264SpeedControl; + +class X264Encoder { +public: + X264Encoder(AVOutputFormat *oformat); // Does not take ownership. + + // Called after the last frame. Will block; once this returns, + // the last data is flushed. + ~X264Encoder(); + + // Must be called before first frame. Does not take ownership. + void add_mux(Mux *mux) { muxes.push_back(mux); } + + // is taken to be raw NV12 data of WIDTHxHEIGHT resolution. + // Does not block. + void add_frame(int64_t pts, int64_t duration, movit::YCbCrLumaCoefficients ycbcr_coefficients, const uint8_t *data, const ReceivedTimestamps &received_ts); + + std::string get_global_headers() const { + while (!x264_init_done) { + sched_yield(); + } + return global_headers; + } + + void change_bitrate(unsigned rate_kbit) { + new_bitrate_kbit = rate_kbit; + } + +private: + struct QueuedFrame { + int64_t pts, duration; + movit::YCbCrLumaCoefficients ycbcr_coefficients; + uint8_t *data; + ReceivedTimestamps received_ts; + }; + void encoder_thread_func(); + void init_x264(); + void encode_frame(QueuedFrame qf); + + // bitrate_kbit can be 0 for no change. + static void speed_control_override_func(unsigned bitrate_kbit, movit::YCbCrLumaCoefficients coefficients, x264_param_t *param); + + // One big memory chunk of all 50 (or whatever) frames, allocated in + // the constructor. All data functions just use pointers into this + // pool. + std::unique_ptr frame_pool; + + std::vector muxes; + bool wants_global_headers; + + std::string global_headers; + std::string buffered_sei; // Will be output before first frame, if any. + + std::thread encoder_thread; + std::atomic x264_init_done{false}; + std::atomic should_quit{false}; + X264Dynamic dyn; + x264_t *x264; + std::unique_ptr speed_control; + + std::atomic new_bitrate_kbit{0}; // 0 for no change. + + // Protects everything below it. + std::mutex mu; + + // Frames that are not being encoded or waiting to be encoded, + // so that add_frame() can use new ones. + std::queue free_frames; + + // Frames that are waiting to be encoded (ie., add_frame() has been + // called, but they are not picked up for encoding yet). + std::queue queued_frames; + + // Whenever the state of changes. + std::condition_variable queued_frames_nonempty; + + // Key is the pts of the frame. + std::unordered_map frames_being_encoded; +}; + +#endif // !defined(_X264ENCODE_H) diff --git a/nageru/x264_speed_control.cpp b/nageru/x264_speed_control.cpp new file mode 100644 index 0000000..719cf28 --- /dev/null +++ b/nageru/x264_speed_control.cpp @@ -0,0 +1,335 @@ +#include "x264_speed_control.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flags.h" +#include "metrics.h" + +using namespace std; +using namespace std::chrono; + +#define SC_PRESETS 23 + +X264SpeedControl::X264SpeedControl(x264_t *x264, float f_speed, int i_buffer_size, float f_buffer_init) + : dyn(load_x264_for_bit_depth(global_flags.x264_bit_depth)), + x264(x264), f_speed(f_speed) +{ + x264_param_t param; + dyn.x264_encoder_parameters(x264, ¶m); + + float fps = (float)param.i_fps_num / param.i_fps_den; + uspf = 1e6 / fps; + set_buffer_size(i_buffer_size); + buffer_fill = buffer_size * f_buffer_init; + buffer_fill = max(buffer_fill, uspf); + buffer_fill = min(buffer_fill, buffer_size); + timestamp = steady_clock::now(); + preset = -1; + cplx_num = 3e3; //FIXME estimate initial complexity + cplx_den = .1; + stat.min_buffer = buffer_size; + stat.max_buffer = 0; + stat.avg_preset = 0.0; + stat.den = 0; + + metric_x264_speedcontrol_buffer_available_seconds = buffer_fill * 1e-6; + metric_x264_speedcontrol_buffer_size_seconds = buffer_size * 1e-6; + metric_x264_speedcontrol_preset_used_frames.init_uniform(SC_PRESETS); + global_metrics.add("x264_speedcontrol_preset_used_frames", &metric_x264_speedcontrol_preset_used_frames); + global_metrics.add("x264_speedcontrol_buffer_available_seconds", &metric_x264_speedcontrol_buffer_available_seconds, Metrics::TYPE_GAUGE); + global_metrics.add("x264_speedcontrol_buffer_size_seconds", &metric_x264_speedcontrol_buffer_size_seconds, Metrics::TYPE_GAUGE); + global_metrics.add("x264_speedcontrol_idle_frames", &metric_x264_speedcontrol_idle_frames); + global_metrics.add("x264_speedcontrol_late_frames", &metric_x264_speedcontrol_late_frames); +} + +X264SpeedControl::~X264SpeedControl() +{ + fprintf(stderr, "speedcontrol: avg preset=%.3f buffer min=%.3f max=%.3f\n", + stat.avg_preset / stat.den, + (float)stat.min_buffer / buffer_size, + (float)stat.max_buffer / buffer_size ); + // x264_log( x264, X264_LOG_INFO, "speedcontrol: avg cplx=%.5f\n", cplx_num / cplx_den ); + if (dyn.handle) { + dlclose(dyn.handle); + } +} + +typedef struct +{ + float time; // relative encoding time, compared to the other presets + int subme; + int me; + int refs; + int mix; + int trellis; + int partitions; + int direct; + int merange; +} sc_preset_t; + +// The actual presets, including the equivalent commandline options. Note that +// all presets are benchmarked with --weightp 1 --mbtree --rc-lookahead 20 +// --b-adapt 1 --bframes 3 on top of the given settings (equivalent settings to +// the "faster" preset). Timings and SSIM measurements were done on a four cores +// of a 6-core Coffee Lake i5 2.8 GHz on the first 1000 frames of “Elephants +// Dream” in 1080p. See experiments/measure-x264.pl for a way to reproduce. +// +// Note that the two first and the two last are also used for extrapolation +// should the desired time be outside the range. Thus, it is disadvantageous if +// they are chosen so that the timings are too close to each other. +static const sc_preset_t presets[SC_PRESETS] = { +#define I4 X264_ANALYSE_I4x4 +#define I8 X264_ANALYSE_I8x8 +#define P4 X264_ANALYSE_PSUB8x8 +#define P8 X264_ANALYSE_PSUB16x16 +#define B8 X264_ANALYSE_BSUB16x16 + + // Preset 0: 17.386db, --preset superfast + { .time= 1.000, .subme=1, .me=X264_ME_DIA, .refs=1, .mix=0, .trellis=0, .partitions=I8|I4, .direct=1, .merange=16 }, + + // Preset 1: 17.919db, --preset superfast --subme 2 + { .time= 1.707, .subme=2, .me=X264_ME_DIA, .refs=1, .mix=0, .trellis=0, .partitions=I8|I4, .direct=1, .merange=16 }, + + // Preset 2: 18.051db, --preset veryfast + { .time= 1.832, .subme=2, .me=X264_ME_HEX, .refs=1, .mix=0, .trellis=0, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 3: 18.422db, --preset veryfast --subme 3 + { .time= 1.853, .subme=3, .me=X264_ME_HEX, .refs=1, .mix=0, .trellis=0, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 4: 18.514db, --preset veryfast --subme 3 --ref 2 + { .time= 1.925, .subme=3, .me=X264_ME_HEX, .refs=2, .mix=0, .trellis=0, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 5: 18.564db, --preset veryfast --subme 4 --ref 2 + { .time= 2.111, .subme=4, .me=X264_ME_HEX, .refs=2, .mix=0, .trellis=0, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 6: 18.411db, --preset faster + { .time= 2.240, .subme=4, .me=X264_ME_HEX, .refs=2, .mix=0, .trellis=1, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 7: 18.429db, --preset faster --mixed-refs + { .time= 2.414, .subme=4, .me=X264_ME_HEX, .refs=2, .mix=1, .trellis=1, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 8: 18.454db, --preset faster --mixed-refs --subme 5 + { .time= 2.888, .subme=5, .me=X264_ME_HEX, .refs=2, .mix=1, .trellis=1, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 9: 18.528db, --preset fast + { .time= 3.570, .subme=6, .me=X264_ME_HEX, .refs=2, .mix=1, .trellis=1, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 10: 18.762db, --preset fast --subme 7 + { .time= 3.698, .subme=7, .me=X264_ME_HEX, .refs=2, .mix=1, .trellis=1, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 11: 18.819db, --preset medium + { .time= 4.174, .subme=7, .me=X264_ME_HEX, .refs=3, .mix=1, .trellis=1, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 12: 18.889db, --preset medium --subme 8 + { .time= 5.155, .subme=8, .me=X264_ME_HEX, .refs=3, .mix=1, .trellis=1, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 13: 19.127db, --preset medium --subme 8 --trellis 2 + { .time= 7.237, .subme=8, .me=X264_ME_HEX, .refs=3, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8, .direct=1, .merange=16 }, + + // Preset 14: 19.118db, --preset medium --subme 8 --trellis 2 --direct auto + { .time= 7.240, .subme=8, .me=X264_ME_HEX, .refs=3, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8, .direct=3, .merange=16 }, + + // Preset 15: 19.172db, --preset slow + { .time= 7.910, .subme=8, .me=X264_ME_HEX, .refs=5, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8, .direct=3, .merange=16 }, + + // Preset 16: 19.208db, --preset slow --subme 9 + { .time= 8.091, .subme=9, .me=X264_ME_HEX, .refs=5, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8, .direct=3, .merange=16 }, + + // Preset 17: 19.216db, --preset slow --subme 9 --me umh + { .time= 9.539, .subme=9, .me=X264_ME_UMH, .refs=5, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8, .direct=3, .merange=16 }, + + // Preset 18: 19.253db, --preset slow --subme 9 --me umh --ref 6 + { .time=10.521, .subme=9, .me=X264_ME_UMH, .refs=6, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8, .direct=3, .merange=16 }, + + // Preset 19: 19.275db, --preset slow --subme 9 --me umh --ref 7 + { .time=11.461, .subme=9, .me=X264_ME_UMH, .refs=7, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8, .direct=3, .merange=16 }, + + // Preset 20: 19.314db, --preset slower + { .time=13.145, .subme=9, .me=X264_ME_UMH, .refs=8, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8|P4, .direct=3, .merange=16 }, + + // Preset 21: 19.407db, --preset slower --subme 10 + { .time=16.386, .subme=10, .me=X264_ME_UMH, .refs=8, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8|P4, .direct=3, .merange=16 }, + + // Preset 22: 19.483db, --preset veryslow + { .time=26.861, .subme=10, .me=X264_ME_UMH, .refs=16, .mix=1, .trellis=2, .partitions=I8|I4|P8|B8|P4, .direct=3, .merange=24 }, + +#undef I4 +#undef I8 +#undef P4 +#undef P8 +#undef B8 +}; + +void X264SpeedControl::before_frame(float new_buffer_fill, int new_buffer_size, float new_uspf) +{ + if (new_uspf > 0.0) { + uspf = new_uspf; + } + if (new_buffer_size) { + set_buffer_size(new_buffer_size); + } + buffer_fill = buffer_size * new_buffer_fill; + metric_x264_speedcontrol_buffer_available_seconds = buffer_fill * 1e-6; + + steady_clock::time_point t; + + // update buffer state after encoding and outputting the previous frame(s) + if (first) { + t = timestamp = steady_clock::now(); + first = false; + } else { + t = steady_clock::now(); + } + + auto delta_t = t - timestamp; + timestamp = t; + + // update the time predictor + if (preset >= 0) { + int cpu_time = duration_cast(cpu_time_last_frame).count(); + cplx_num *= cplx_decay; + cplx_den *= cplx_decay; + cplx_num += cpu_time / presets[preset].time; + ++cplx_den; + + stat.avg_preset += preset; + ++stat.den; + } + + stat.min_buffer = min(buffer_fill, stat.min_buffer); + stat.max_buffer = max(buffer_fill, stat.max_buffer); + + if (buffer_fill >= buffer_size) { // oops, cpu was idle + // not really an error, but we'll warn for debugging purposes + static int64_t idle_t = 0; + static steady_clock::time_point print_interval; + static bool first = false; + idle_t += buffer_fill - buffer_size; + if (first || duration(t - print_interval).count() > 0.1) { + //fprintf(stderr, "speedcontrol idle (%.6f sec)\n", idle_t/1e6); + print_interval = t; + idle_t = 0; + first = false; + } + buffer_fill = buffer_size; + metric_x264_speedcontrol_buffer_available_seconds = buffer_fill * 1e-6; + ++metric_x264_speedcontrol_idle_frames; + } else if (buffer_fill <= 0) { // oops, we're late + // fprintf(stderr, "speedcontrol underflow (%.6f sec)\n", buffer_fill/1e6); + ++metric_x264_speedcontrol_late_frames; + } + + { + // Pick the preset that should return the buffer to 3/4-full within a time + // specified by compensation_period. + // + // NOTE: This doesn't actually do that, at least assuming the same target is + // chosen for every frame; exactly what it does is unclear to me. It seems + // to consistently undershoot a bit, so it needs to be saved by the second + // predictor below. However, fixing the formula seems to yield somewhat less + // stable results in practice; in particular, once the buffer is half-full + // or so, it would give us a negative target. Perhaps increasing + // compensation_period would be a good idea, but initial (very brief) tests + // did not yield good results. + float target = uspf / f_speed + * (buffer_fill + compensation_period) + / (buffer_size*3/4 + compensation_period); + float cplx = cplx_num / cplx_den; + float set, t0, t1; + float filled = (float) buffer_fill / buffer_size; + int i; + t0 = presets[0].time * cplx; + for (i = 1; ; i++) { + t1 = presets[i].time * cplx; + if (t1 >= target || i == SC_PRESETS - 1) + break; + t0 = t1; + } + // exponential interpolation between states + set = i-1 + (log(target) - log(t0)) / (log(t1) - log(t0)); + set = max(set, -5); + set = min(set, (SC_PRESETS-1) + 5); + // Even if our time estimations in the SC_PRESETS array are off + // this will push us towards our target fullness + float s1 = set; + set += (40 * (filled-0.75)); + float s2 = (40 * (filled-0.75)); + set = min(max(set, 0), SC_PRESETS - 1); + apply_preset(dither_preset(set)); + + if (global_flags.x264_speedcontrol_verbose) { + static float cpu, wall, tgt, den; + const float decay = 1-1/100.; + cpu = cpu*decay + duration_cast(cpu_time_last_frame).count(); + wall = wall*decay + duration_cast(delta_t).count(); + tgt = tgt*decay + target; + den = den*decay + 1; + fprintf(stderr, "speed: %.2f+%.2f %d[%.5f] (t/c/w: %6.0f/%6.0f/%6.0f = %.4f) fps=%.2f\r", + s1, s2, preset, (float)buffer_fill / buffer_size, + tgt/den, cpu/den, wall/den, cpu/wall, 1e6*den/wall ); + } + } + +} + +void X264SpeedControl::after_frame() +{ + cpu_time_last_frame = steady_clock::now() - timestamp; +} + +void X264SpeedControl::set_buffer_size(int new_buffer_size) +{ + new_buffer_size = max(3, new_buffer_size); + buffer_size = new_buffer_size * uspf; + cplx_decay = 1 - 1./new_buffer_size; + compensation_period = buffer_size/4; + metric_x264_speedcontrol_buffer_size_seconds = buffer_size * 1e-6; +} + +int X264SpeedControl::dither_preset(float f) +{ + int i = f; + if (f < 0) { + i--; + } + dither += f - i; + if (dither >= 1.0) { + dither--; + i++; + } + return i; +} + +void X264SpeedControl::apply_preset(int new_preset) +{ + new_preset = max(new_preset, 0); + new_preset = min(new_preset, SC_PRESETS - 1); + + const sc_preset_t *s = &presets[new_preset]; + x264_param_t p; + dyn.x264_encoder_parameters(x264, &p); + + p.i_frame_reference = s->refs; + p.analyse.inter = s->partitions; + p.analyse.i_subpel_refine = s->subme; + p.analyse.i_me_method = s->me; + p.analyse.i_trellis = s->trellis; + p.analyse.b_mixed_references = s->mix; + p.analyse.i_direct_mv_pred = s->direct; + p.analyse.i_me_range = s->merange; + if (override_func) { + override_func(&p); + } + dyn.x264_encoder_reconfig(x264, &p); + preset = new_preset; + + metric_x264_speedcontrol_preset_used_frames.count_event(new_preset); +} diff --git a/nageru/x264_speed_control.h b/nageru/x264_speed_control.h new file mode 100644 index 0000000..b0a1739 --- /dev/null +++ b/nageru/x264_speed_control.h @@ -0,0 +1,144 @@ +#ifndef _X264_SPEED_CONTROL_H +#define _X264_SPEED_CONTROL_H 1 + +// The x264 speed control tries to encode video at maximum possible quality +// without skipping frames (at the expense of higher encoding latency and +// less even output rates, although VBV is still respected). It does this +// by continuously (every frame) changing the x264 quality settings such that +// it uses maximum amount of CPU, but no more. +// +// Speed control works by maintaining a queue of frames, with the confusing +// nomenclature “full” meaning that there are no queues in the frame. +// (Conversely, if the queue is “empty” and a new frame comes in, we need to +// drop that frame.) It tries to keep the buffer 3/4 “full” by using a table +// of measured relative speeds for the different presets, and choosing one that it +// thinks will return the buffer to that state over time. However, since +// different frames take different times to encode regardless of preset, it +// also tries to maintain a running average of how long the typical frame will +// take to encode at the fastest preset (the so-called “complexity”), by dividing +// the actual time by the relative time for the preset used. +// +// Frame timings is a complex topic in its own sright, since usually, multiple +// frames are encoded in parallel. X264SpeedControl only supports the timing +// method that the original patch calls “alternate timing”; one simply measures +// the time the last x264_encoder_encode() call took. (The other alternative given +// is to measure the time between successive x264_encoder_encode() calls.) +// Unless using the zerocopy presets (which activate slice threading), the function +// actually returns not when the given frame is done encoding, but when one a few +// frames back is done encoding. So it doesn't actually measure the time of any +// given one frame, but it measures something correlated to it, at least as long as +// you are near 100% CPU utilization (ie., the encoded frame doesn't linger in the +// buffers already when x264_encoder_encode() is called). +// +// The code has a long history; it was originally part of Avail Media's x264 +// branch, used in their encoder appliances, and then a snapshot of that was +// released. (Given that x264 is licensed under GPLv2 or newer, this means that +// we can also treat the patch as GPLv2 or newer if we want, which we do. +// As far as I know, it is copyright Avail Media, although no specific copyright +// notice was posted on the patch.) +// +// From there, it was incorporated in OBE's x264 tree (x264-obe) and some bugs +// were fixed. I started working on it for the purposes of Nageru, fixing various +// issues, adding VFR support and redoing the timings entirely based on more +// modern presets (the patch was made before several important x264 features, +// such as weighted P-frames). Finally, I took it out of x264 and put it into +// Nageru (it does not actually use any hooks into the codec itself), so that +// one does not need to patch x264 to use it in Nageru. It still could do with +// some cleanup, but it's much, much better than just using a static preset. + +#include +#include +#include +#include + +extern "C" { +#include +} + +#include "metrics.h" +#include "x264_dynamic.h" + +class X264SpeedControl { +public: + // x264: Encoding object we are using; must be opened. Assumed to be + // set to the "faster" preset, and with 16 reference frames. + // f_speed: Relative encoding speed, usually 1.0. + // i_buffer_size: Number of frames in the buffer. + // f_buffer_init: Relative fullness of buffer at start + // (0.0 = assumed to be frames in buffer, + // 1.0 = no frames in buffer) + X264SpeedControl(x264_t *x264, float f_speed, int i_buffer_size, float f_buffer_init); + ~X264SpeedControl(); + + // You need to call before_frame() immediately before each call to + // x264_encoder_encode(), and after_frame() immediately after. + // + // new_buffer_fill: Buffer fullness, in microseconds (_not_ a relative + // number, unlike f_buffer_init in the constructor). + // new_buffer_size: If > 0, new number of frames in the buffer, + // ie. the buffer size has changed. (It is harmless to set this + // even if the buffer hasn't actually changed.) + // f_uspf: If > 0, new microseconds per frame, ie. the frame rate has + // changed. (Of course, with VFR, it can be impossible to truly know + // the frame rate of the coming frames, but it is a reasonable + // assumption that the next second or so is likely to be the same + // frame rate as the last frame.) + void before_frame(float new_buffer_fill, int new_buffer_size, float f_uspf); + void after_frame(); + + // x264 seemingly has an issue where x264_encoder_reconfig() is not reflected + // immediately in x264_encoder_parameters(). Since speed control keeps calling + // those two all the time, any changes you make outside X264SpeedControl + // could be overridden. Thus, to make changes to encoder parameters, you should + // instead set a function here, which will be called every time parameters + // are modified. + void set_config_override_function(std::function override_func) + { + this->override_func = override_func; + } + +private: + void set_buffer_size(int new_buffer_size); + int dither_preset(float f); + void apply_preset(int new_preset); + + X264Dynamic dyn; + + // Not owned by us. + x264_t *x264; + + float f_speed; + + // all times that are not std::chrono::* are in usec + std::chrono::steady_clock::time_point timestamp; // when was speedcontrol last invoked + std::chrono::steady_clock::duration cpu_time_last_frame{std::chrono::seconds{0}}; // time spent encoding the previous frame + int64_t buffer_size; // assumed application-side buffer of frames to be streamed (measured in microseconds), + int64_t buffer_fill; // where full = we don't have to hurry + int64_t compensation_period; // how quickly we try to return to the target buffer fullness + float uspf; // microseconds per frame + int preset = -1; // which setting was used in the previous frame + float cplx_num = 3e3; // rolling average of estimated spf for preset #0. FIXME estimate initial complexity + float cplx_den = .1; + float cplx_decay; + float dither = 0.0f; + + bool first = true; + + struct + { + int64_t min_buffer, max_buffer; + double avg_preset; + int den; + } stat; + + std::function override_func = nullptr; + + // Metrics. + Histogram metric_x264_speedcontrol_preset_used_frames; + std::atomic metric_x264_speedcontrol_buffer_available_seconds{0.0}; + std::atomic metric_x264_speedcontrol_buffer_size_seconds{0.0}; + std::atomic metric_x264_speedcontrol_idle_frames{0}; + std::atomic metric_x264_speedcontrol_late_frames{0}; +}; + +#endif // !defined(_X264_SPEED_CONTROL_H) diff --git a/nageru/ycbcr_interpretation.h b/nageru/ycbcr_interpretation.h new file mode 100644 index 0000000..51bad76 --- /dev/null +++ b/nageru/ycbcr_interpretation.h @@ -0,0 +1,12 @@ +#ifndef _YCBCR_INTERPRETATION_H +#define _YCBCR_INTERPRETATION_H 1 + +#include + +struct YCbCrInterpretation { + bool ycbcr_coefficients_auto = true; + movit::YCbCrLumaCoefficients ycbcr_coefficients = movit::YCBCR_REC_709; + bool full_range = false; +}; + +#endif // !defined(_YCBCR_INTERPRETATION_H) diff --git a/patches/zita-resampler-sse.diff b/patches/zita-resampler-sse.diff new file mode 100644 index 0000000..4954515 --- /dev/null +++ b/patches/zita-resampler-sse.diff @@ -0,0 +1,417 @@ +diff -ur orig/zita-resampler-1.3.0/libs/resampler.cc zita-resampler-1.3.0/libs/resampler.cc +--- orig/zita-resampler-1.3.0/libs/resampler.cc 2012-10-26 22:58:55.000000000 +0200 ++++ zita-resampler-1.3.0/libs/resampler.cc 2016-09-05 00:30:34.520191288 +0200 +@@ -24,6 +24,10 @@ + #include + #include + ++#ifdef __SSE2__ ++#include ++#endif ++ + + static unsigned int gcd (unsigned int a, unsigned int b) + { +@@ -47,6 +51,118 @@ + return 1; + } + ++#ifdef __SSE2__ ++ ++static inline float calc_mono_sample_sse (unsigned int hl, ++ const float *c1, ++ const float *c2, ++ const float *q1, ++ const float *q2) ++{ ++ unsigned int i; ++ __m128 denorm, s, w1, w2, shuf; ++ ++ denorm = _mm_set1_ps (1e-20f); ++ s = denorm; ++ for (i = 0; i < hl; i += 4) ++ { ++ q2 -= 4; ++ ++ // s += *q1 * c1 [i]; ++ w1 = _mm_loadu_ps (&c1 [i]); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q1), w1)); ++ ++ // s += *q2 * c2 [i]; ++ w2 = _mm_loadu_ps (&c2 [i]); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q2), _mm_shuffle_ps (w2, w2, _MM_SHUFFLE (0, 1, 2, 3)))); ++ ++ q1 += 4; ++ } ++ s = _mm_sub_ps (s, denorm); ++ ++ // Add all the elements of s together into one. Adapted from ++ // http://stackoverflow.com/questions/6996764/fastest-way-to-do-horizontal-float-vector-sum-on-x86 ++ shuf = _mm_shuffle_ps (s, s, _MM_SHUFFLE (2, 3, 0, 1)); ++ s = _mm_add_ps (s, shuf); ++ s = _mm_add_ss (s, _mm_movehl_ps (shuf, s)); ++ return _mm_cvtss_f32 (s); ++} ++ ++// Note: This writes four floats instead of two (the last two are garbage). ++// The caller will need to make sure there is room for all four. ++static inline void calc_stereo_sample_sse (unsigned int hl, ++ const float *c1, ++ const float *c2, ++ const float *q1, ++ const float *q2, ++ float *out_data) ++{ ++ unsigned int i; ++ __m128 denorm, s, w1, w2; ++ ++ denorm = _mm_set1_ps (1e-20f); ++ s = denorm; ++ for (i = 0; i < hl; i += 4) ++ { ++ q2 -= 8; ++ ++ // s += *q1 * c1 [i]; ++ w1 = _mm_loadu_ps (&c1 [i]); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q1), _mm_unpacklo_ps (w1, w1))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q1 + 4), _mm_unpackhi_ps (w1, w1))); ++ ++ // s += *q2 * c2 [i]; ++ w2 = _mm_loadu_ps (&c2 [i]); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q2 + 4), _mm_shuffle_ps (w2, w2, _MM_SHUFFLE (0, 0, 1, 1)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q2), _mm_shuffle_ps (w2, w2, _MM_SHUFFLE (2, 2, 3, 3)))); ++ ++ q1 += 8; ++ } ++ s = _mm_sub_ps (s, denorm); ++ s = _mm_add_ps (s, _mm_shuffle_ps (s, s, _MM_SHUFFLE (1, 0, 3, 2))); ++ ++ _mm_storeu_ps (out_data, s); ++} ++ ++static inline void calc_quad_sample_sse (int hl, ++ int nchan, ++ const float *c1, ++ const float *c2, ++ const float *q1, ++ const float *q2, ++ float *out_data) ++{ ++ int i; ++ __m128 denorm, s, w1, w2; ++ ++ denorm = _mm_set1_ps (1e-20f); ++ s = denorm; ++ for (i = 0; i < hl; i += 4) ++ { ++ q2 -= 4 * nchan; ++ ++ // s += *p1 * _c1 [i]; ++ w1 = _mm_loadu_ps (&c1 [i]); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q1), _mm_shuffle_ps (w1, w1, _MM_SHUFFLE (0, 0, 0, 0)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q1 + nchan), _mm_shuffle_ps (w1, w1, _MM_SHUFFLE (1, 1, 1, 1)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q1 + 2 * nchan), _mm_shuffle_ps (w1, w1, _MM_SHUFFLE (2, 2, 2, 2)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q1 + 3 * nchan), _mm_shuffle_ps (w1, w1, _MM_SHUFFLE (3, 3, 3, 3)))); ++ ++ // s += *p2 * _c2 [i]; ++ w2 = _mm_loadu_ps (&c2 [i]); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q2 + 3 * nchan), _mm_shuffle_ps (w2, w2, _MM_SHUFFLE (0, 0, 0, 0)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q2 + 2 * nchan), _mm_shuffle_ps (w2, w2, _MM_SHUFFLE (1, 1, 1, 1)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q2 + nchan), _mm_shuffle_ps (w2, w2, _MM_SHUFFLE (2, 2, 2, 2)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (q2), _mm_shuffle_ps (w2, w2, _MM_SHUFFLE (3, 3, 3, 3)))); ++ ++ q1 += 4 * nchan; ++ } ++ s = _mm_sub_ps (s, denorm); ++ ++ _mm_storeu_ps (out_data, s); ++} ++#endif ++ + + Resampler::Resampler (void) : + _table (0), +@@ -213,18 +329,42 @@ + { + float *c1 = _table->_ctab + hl * ph; + float *c2 = _table->_ctab + hl * (np - ph); +- for (c = 0; c < _nchan; c++) ++#ifdef __SSE2__ ++ if ((hl % 4) == 0 && _nchan == 1) ++ { ++ *out_data++ = calc_mono_sample_sse (hl, c1, c2, p1, p2); ++ } ++ else if ((hl % 4) == 0 && _nchan == 2) + { +- float *q1 = p1 + c; +- float *q2 = p2 + c; +- float s = 1e-20f; +- for (i = 0; i < hl; i++) ++ if (out_count >= 2) ++ { ++ calc_stereo_sample_sse (hl, c1, c2, p1, p2, out_data); ++ } ++ else ++ { ++ float tmp[4]; ++ calc_stereo_sample_sse (hl, c1, c2, p1, p2, tmp); ++ out_data[0] = tmp[0]; ++ out_data[1] = tmp[1]; ++ } ++ out_data += 2; ++ } ++ else ++#endif ++ { ++ for (c = 0; c < _nchan; c++) + { +- q2 -= _nchan; +- s += *q1 * c1 [i] + *q2 * c2 [i]; +- q1 += _nchan; ++ float *q1 = p1 + c; ++ float *q2 = p2 + c; ++ float s = 1e-20f; ++ for (i = 0; i < hl; i++) ++ { ++ q2 -= _nchan; ++ s += *q1 * c1 [i] + *q2 * c2 [i]; ++ q1 += _nchan; ++ } ++ *out_data++ = s - 1e-20f; + } +- *out_data++ = s - 1e-20f; + } + } + else +diff -ur orig/zita-resampler-1.3.0/libs/vresampler.cc zita-resampler-1.3.0/libs/vresampler.cc +--- orig/zita-resampler-1.3.0/libs/vresampler.cc 2012-10-26 22:58:55.000000000 +0200 ++++ zita-resampler-1.3.0/libs/vresampler.cc 2016-09-05 00:33:53.907511211 +0200 +@@ -25,6 +25,152 @@ + #include + + ++#ifdef __SSE2__ ++ ++#include ++ ++static inline float calc_mono_sample_sse (int hl, ++ float b, ++ const float *p1, ++ const float *p2, ++ const float *q1, ++ const float *q2) ++{ ++ int i; ++ __m128 denorm, bs, s, c1, c2, w1, w2, shuf; ++ ++ denorm = _mm_set1_ps (1e-25f); ++ bs = _mm_set1_ps (b); ++ s = denorm; ++ for (i = 0; i < hl; i += 4) ++ { ++ p2 -= 4; ++ ++ // _c1 [i] = q1 [i] + b * (q1 [i + hl] - q1 [i]); ++ w1 = _mm_loadu_ps (&q1 [i]); ++ w2 = _mm_loadu_ps (&q1 [i + hl]); ++ c1 = _mm_add_ps (w1, _mm_mul_ps(bs, _mm_sub_ps (w2, w1))); ++ ++ // _c2 [i] = q2 [i] + b * (q2 [i - hl] - q2 [i]); ++ w1 = _mm_loadu_ps (&q2 [i]); ++ w2 = _mm_loadu_ps (&q2 [i - hl]); ++ c2 = _mm_add_ps (w1, _mm_mul_ps(bs, _mm_sub_ps (w2, w1))); ++ ++ // s += *p1 * _c1 [i]; ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p1), c1)); ++ ++ // s += *p2 * _c2 [i]; ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p2), _mm_shuffle_ps (c2, c2, _MM_SHUFFLE (0, 1, 2, 3)))); ++ ++ p1 += 4; ++ } ++ s = _mm_sub_ps (s, denorm); ++ ++ // Add all the elements of s together into one. Adapted from ++ // http://stackoverflow.com/questions/6996764/fastest-way-to-do-horizontal-float-vector-sum-on-x86 ++ shuf = _mm_shuffle_ps (s, s, _MM_SHUFFLE (2, 3, 0, 1)); ++ s = _mm_add_ps (s, shuf); ++ s = _mm_add_ss (s, _mm_movehl_ps (shuf, s)); ++ return _mm_cvtss_f32 (s); ++} ++ ++// Note: This writes four floats instead of two (the last two are garbage). ++// The caller will need to make sure there is room for all four. ++static inline void calc_stereo_sample_sse (int hl, ++ float b, ++ const float *p1, ++ const float *p2, ++ const float *q1, ++ const float *q2, ++ float *out_data) ++{ ++ int i; ++ __m128 denorm, bs, s, c1, c2, w1, w2; ++ ++ denorm = _mm_set1_ps (1e-25f); ++ bs = _mm_set1_ps (b); ++ s = denorm; ++ for (i = 0; i < hl; i += 4) ++ { ++ p2 -= 8; ++ ++ // _c1 [i] = q1 [i] + b * (q1 [i + hl] - q1 [i]); ++ w1 = _mm_loadu_ps (&q1 [i]); ++ w2 = _mm_loadu_ps (&q1 [i + hl]); ++ c1 = _mm_add_ps (w1, _mm_mul_ps(bs, _mm_sub_ps (w2, w1))); ++ ++ // _c2 [i] = q2 [i] + b * (q2 [i - hl] - q2 [i]); ++ w1 = _mm_loadu_ps (&q2 [i]); ++ w2 = _mm_loadu_ps (&q2 [i - hl]); ++ c2 = _mm_add_ps (w1, _mm_mul_ps(bs, _mm_sub_ps (w2, w1))); ++ ++ // s += *p1 * _c1 [i]; ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p1), _mm_unpacklo_ps (c1, c1))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p1 + 4), _mm_unpackhi_ps (c1, c1))); ++ ++ // s += *p2 * _c2 [i]; ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p2 + 4), _mm_shuffle_ps (c2, c2, _MM_SHUFFLE (0, 0, 1, 1)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p2), _mm_shuffle_ps (c2, c2, _MM_SHUFFLE (2, 2, 3, 3)))); ++ ++ p1 += 8; ++ } ++ s = _mm_sub_ps (s, denorm); ++ s = _mm_add_ps (s, _mm_shuffle_ps (s, s, _MM_SHUFFLE (1, 0, 3, 2))); ++ ++ _mm_storeu_ps (out_data, s); ++} ++ ++static inline void calc_quad_sample_sse (int hl, ++ int nchan, ++ float b, ++ const float *p1, ++ const float *p2, ++ const float *q1, ++ const float *q2, ++ float *out_data) ++{ ++ int i; ++ __m128 denorm, bs, s, c1, c2, w1, w2; ++ ++ denorm = _mm_set1_ps (1e-25f); ++ bs = _mm_set1_ps (b); ++ s = denorm; ++ for (i = 0; i < hl; i += 4) ++ { ++ p2 -= 4 * nchan; ++ ++ // _c1 [i] = q1 [i] + b * (q1 [i + hl] - q1 [i]); ++ w1 = _mm_loadu_ps (&q1 [i]); ++ w2 = _mm_loadu_ps (&q1 [i + hl]); ++ c1 = _mm_add_ps (w1, _mm_mul_ps(bs, _mm_sub_ps (w2, w1))); ++ ++ // _c2 [i] = q2 [i] + b * (q2 [i - hl] - q2 [i]); ++ w1 = _mm_loadu_ps (&q2 [i]); ++ w2 = _mm_loadu_ps (&q2 [i - hl]); ++ c2 = _mm_add_ps (w1, _mm_mul_ps(bs, _mm_sub_ps (w2, w1))); ++ ++ // s += *p1 * _c1 [i]; ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p1), _mm_shuffle_ps (c1, c1, _MM_SHUFFLE (0, 0, 0, 0)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p1 + nchan), _mm_shuffle_ps (c1, c1, _MM_SHUFFLE (1, 1, 1, 1)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p1 + 2 * nchan), _mm_shuffle_ps (c1, c1, _MM_SHUFFLE (2, 2, 2, 2)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p1 + 3 * nchan), _mm_shuffle_ps (c1, c1, _MM_SHUFFLE (3, 3, 3, 3)))); ++ ++ // s += *p2 * _c2 [i]; ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p2 + 3 * nchan), _mm_shuffle_ps (c2, c2, _MM_SHUFFLE (0, 0, 0, 0)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p2 + 2 * nchan), _mm_shuffle_ps (c2, c2, _MM_SHUFFLE (1, 1, 1, 1)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p2 + nchan), _mm_shuffle_ps (c2, c2, _MM_SHUFFLE (2, 2, 2, 2)))); ++ s = _mm_add_ps (s, _mm_mul_ps (_mm_loadu_ps (p2), _mm_shuffle_ps (c2, c2, _MM_SHUFFLE (3, 3, 3, 3)))); ++ ++ p1 += 4 * nchan; ++ } ++ s = _mm_sub_ps (s, denorm); ++ ++ _mm_storeu_ps (out_data, s); ++} ++ ++#endif ++ ++ + VResampler::VResampler (void) : + _table (0), + _nchan (0), +@@ -163,7 +309,7 @@ + + int VResampler::process (void) + { +- unsigned int k, np, in, nr, n, c; ++ unsigned int j, k, np, in, nr, n, c; + int i, hl, nz; + double ph, dp, dd; + float a, b, *p1, *p2, *q1, *q2; +@@ -212,23 +358,55 @@ + a = 1.0f - b; + q1 = _table->_ctab + hl * k; + q2 = _table->_ctab + hl * (np - k); +- for (i = 0; i < hl; i++) ++#ifdef __SSE2__ ++ if ((hl % 4) == 0 && _nchan == 1) ++ { ++ *out_data++ = calc_mono_sample_sse (hl, b, p1, p2, q1, q2); ++ } ++ else if ((hl % 4) == 0 && _nchan == 2) + { +- _c1 [i] = a * q1 [i] + b * q1 [i + hl]; +- _c2 [i] = a * q2 [i] + b * q2 [i - hl]; ++ if (out_count >= 2) ++ { ++ calc_stereo_sample_sse (hl, b, p1, p2, q1, q2, out_data); ++ } ++ else ++ { ++ float tmp[4]; ++ calc_stereo_sample_sse (hl, b, p1, p2, q1, q2, tmp); ++ out_data[0] = tmp[0]; ++ out_data[1] = tmp[1]; ++ } ++ out_data += 2; ++ } ++ else if ((hl % 4) == 0 && (_nchan % 4) == 0) ++ { ++ for (j = 0; j < _nchan; j += 4) ++ { ++ calc_quad_sample_sse (hl, _nchan, b, p1 + j, p2 + j, q1, q2, out_data + j); ++ } ++ out_data += _nchan; + } +- for (c = 0; c < _nchan; c++) ++ else ++#endif + { +- q1 = p1 + c; +- q2 = p2 + c; +- a = 1e-25f; + for (i = 0; i < hl; i++) + { +- q2 -= _nchan; +- a += *q1 * _c1 [i] + *q2 * _c2 [i]; +- q1 += _nchan; ++ _c1 [i] = a * q1 [i] + b * q1 [i + hl]; ++ _c2 [i] = a * q2 [i] + b * q2 [i - hl]; ++ } ++ for (c = 0; c < _nchan; c++) ++ { ++ q1 = p1 + c; ++ q2 = p2 + c; ++ a = 1e-25f; ++ for (i = 0; i < hl; i++) ++ { ++ q2 -= _nchan; ++ a += *q1 * _c1 [i] + *q2 * _c2 [i]; ++ q1 += _nchan; ++ } ++ *out_data++ = a - 1e-25f; + } +- *out_data++ = a - 1e-25f; + } + } + else diff --git a/ref.raw b/ref.raw new file mode 100644 index 0000000..210b8c3 Binary files /dev/null and b/ref.raw differ